Mercurial > prosody-modules

annotate mod_storage_xmlarchive/mod_storage_xmlarchive.lua @ 5551:8bfcedd93a72

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mod_rest: List all error conditions in OpenAPI spec These are not handled by datamanager but by util.stanza and util.error, so they are not represented in the JSON schema file.
author Kim Alvefur <zash@zash.se>
date Sat, 17 Jun 2023 16:26:33 +0200
parents 98bf0f597df4
children 79ba1a1a75cc
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1764
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
1 -- mod_storage_xmlarchive
4647
b91e40472d68 mod_storage_xmlarchive: Fix return of numeric 'when' from key-value API
Kim Alvefur <zash@zash.se>
parents: 4646
diff changeset
2 -- Copyright (C) 2015-2021 Kim Alvefur
1764
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
3 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
4 -- This file is MIT/X11 licensed.
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
5 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
6 -- luacheck: ignore unused self
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
7
2290
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
8 local lfs = require "lfs";
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
9 local dm = require "core.storagemanager".olddm;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
10 local hmac_sha256 = require"util.hashes".hmac_sha256;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
11 local st = require"util.stanza";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
12 local dt = require"util.datetime";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
13 local new_stream = require "util.xmppstream".new;
3898
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
14 local xml = require "util.xml";
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
15 local async = require "util.async";
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
16 local empty = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
17
2289
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
18 if not dm.append_raw then
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
19 module:require"datamanager_append_raw";
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
20 end
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
21
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
22 local archive = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
23 local archive_mt = { __index = archive };
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
24
4646
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm)
Kim Alvefur <zash@zash.se>
parents: 4551
diff changeset
25 archive.caps = {
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm)
Kim Alvefur <zash@zash.se>
parents: 4551
diff changeset
26 total = false,
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm)
Kim Alvefur <zash@zash.se>
parents: 4551
diff changeset
27 quota = nil,
4648
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
28 full_id_range = true;
4649
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
29 ids = true;
4646
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm)
Kim Alvefur <zash@zash.se>
parents: 4551
diff changeset
30 };
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm)
Kim Alvefur <zash@zash.se>
parents: 4551
diff changeset
31
2420
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available
Kim Alvefur <zash@zash.se>
parents: 2405
diff changeset
32 local is_stanza = st.is_stanza or function (s)
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available
Kim Alvefur <zash@zash.se>
parents: 2405
diff changeset
33 return getmetatable(s) == st.stanza_mt;
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available
Kim Alvefur <zash@zash.se>
parents: 2405
diff changeset
34 end
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available
Kim Alvefur <zash@zash.se>
parents: 2405
diff changeset
35
1753
54c8a0cb2996 mod_storage_(archive-capable): Change order of arguments to :append to be the same as return values from :find iterator (see prosody 41725f3df3cc)
Kim Alvefur <zash@zash.se>
parents: 1752
diff changeset
36 function archive:append(username, _, data, when, with)
2420
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available
Kim Alvefur <zash@zash.se>
parents: 2405
diff changeset
37 if not is_stanza(data) then
1739
940a4ab75cec mod_storage_xmlarchive: This module only supports storage of stanzas, log a warning about attempts to do otherwise
Kim Alvefur <zash@zash.se>
parents: 1738
diff changeset
38 module:log("error", "Attempt to store non-stanza object, traceback: %s", debug.traceback());
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
39 return nil, "unsupported-datatype";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
40 end
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
41
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
42 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
43 data = tostring(data) .. "\n";
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
44
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
45 local day = dt.date(when);
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
46 local ok, err = dm.append_raw(username.."@"..day, self.host, self.store, "xml", data);
1969
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
47 if not ok then
3755
bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't
Kim Alvefur <zash@zash.se>
parents: 3587
diff changeset
48 -- append_raw, unlike list_append, does not log anything on failure atm, so we do so here
bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't
Kim Alvefur <zash@zash.se>
parents: 3587
diff changeset
49 module:log("error", "Unable to write to %s storage ('%s') for user: %s@%s",
bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't
Kim Alvefur <zash@zash.se>
parents: 3587
diff changeset
50 self.store, err, username, module.host)
1969
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
51 return nil, err;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
52 end
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
53
2290
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
54 -- If the day-file is missing then we need to add it to the list of days
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
55 local first_of_day = not lfs.attributes(dm.getpath(username .. "@" .. day, self.host, self.store, "list"));
2290
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
56
2595
307ddebb72e1 mod_storage_xmlarchive: Assume offset to be zero if not included (thanks pep.)
Kim Alvefur <zash@zash.se>
parents: 2514
diff changeset
57 local offset = ok and err or 0;
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
58
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
59 local id = day .. "-" .. hmac_sha256(username.."@"..day.."+"..offset, data, true):sub(-16);
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
60 ok, err = dm.list_append(username.."@"..day, self.host, self.store,
2600
9ec8289f6bb2 mod_storage_xmlarchive: Break long line [luacheck]
Kim Alvefur <zash@zash.se>
parents: 2595
diff changeset
61 { id = id, when = dt.datetime(when), with = with, offset = offset, length = #data });
2290
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
62 if ok and first_of_day then
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
63 ok, err = dm.list_append(username, self.host, self.store, day);
1969
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
64 end
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
65 if not ok then
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
66 return nil, err;
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
67 end
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
68 return id;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
69 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
70
3898
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
71 function archive:get(username, id)
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
72 local dates = self:dates(username) or empty;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
73 local day_idx, item_idx, items = self:_get_idx(username, id, dates);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
74 if not day_idx then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
75 return nil, "item-not-found";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
76 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
77 module:log("debug", ":_get_idx(%q, %q) --> %q, %q", username, id, dates[day_idx], items[item_idx]);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
78 local day = dates[day_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
79 local item = items[item_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
80 module:log("debug", "item = %q", item);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
81 local filename = dm.getpath(username.."@"..day, self.host, self.store, "xml");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
82 local xmlfile, ferr = io.open(filename, "r");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
83 if not xmlfile then return nil, ferr; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
84 local p,err = xmlfile:seek("set", item.offset);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
85 if p ~= item.offset or err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
86 local data = xmlfile:read(item.length);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
87 local parsed, perr = xml.parse(data);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
88 if not parsed then return nil, perr; end
4647
b91e40472d68 mod_storage_xmlarchive: Fix return of numeric 'when' from key-value API
Kim Alvefur <zash@zash.se>
parents: 4646
diff changeset
89 return parsed, tonumber(item.when) or dt.parse(item.when), item.with;
3898
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
90 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
91
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
92 local overwrite = module:get_option("xmlarchive_overwrite", false);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
93
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
94 function archive:set(username, id, data, new_when, new_with)
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
95 if not is_stanza(data) then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
96 module:log("error", "Attempt to store non-stanza object, traceback: %s", debug.traceback());
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
97 return nil, "unsupported-datatype";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
98 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
99
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
100 username = username or "@";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
101 data = tostring(data) .. "\n";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
102
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
103 local dates = self:dates(username) or empty;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
104 local day_idx, item_idx, items = self:_get_idx(username, id, dates);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
105 if not day_idx then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
106 return nil, "item-not-found";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
107 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
108 local day = dates[day_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
109 local item = items[item_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
110
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
111 local filename = dm.getpath(username.."@"..day, self.host, self.store, "xml");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
112
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
113 local replaced, err = dm.append_raw(username.."@"..day, self.host, self.store, "xml", data);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
114 if not replaced then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
115 local new_offset = err;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
116
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
117 -- default yes or no?
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
118 if overwrite then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
119 local xmlfile, ferr = io.open(filename, "r+");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
120 if not xmlfile then return nil, ferr; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
121 local p,err = xmlfile:seek("set", item.offset);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
122 if p ~= item.offset or err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
123 local _,err = xmlfile:write((" "):rep(item.length));
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
124 if err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
125 local _,err = xmlfile:close();
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
126 if err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
127 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
128
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
129 items[item_idx] = {
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
130 id = id,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
131 when = new_when or item.when,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
132 with = new_with or item.with,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
133 offset = new_offset,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
134 length = #data,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
135 replaces = item,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
136 };
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
137 local ok, err = dm.list_store(username.."@"..day, self.host, self.store, items);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
138 return ok, err;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
139 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
140
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
141 local function get_nexttick()
4551
b92147edd172 mod_storage_xmlarchive: Workaround for #1646 (util.async bug with Lua 5.1)
Kim Alvefur <zash@zash.se>
parents: 4538
diff changeset
142 -- COMPAT util.async under Lua 5.1 runs into problems trying to yield across
b92147edd172 mod_storage_xmlarchive: Workaround for #1646 (util.async bug with Lua 5.1)
Kim Alvefur <zash@zash.se>
parents: 4538
diff changeset
143 -- pcall barriers. Workaround for #1646
b92147edd172 mod_storage_xmlarchive: Workaround for #1646 (util.async bug with Lua 5.1)
Kim Alvefur <zash@zash.se>
parents: 4538
diff changeset
144 if _VERSION ~= "Lua 5.1" and async.ready() then
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
145 return function ()
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
146 -- slow down
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
147 local wait, done = async.waiter();
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
148 module:add_timer(0, done);
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
149 wait();
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
150 end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
151 else
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
152 -- no async, no-op
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
153 return function () end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
154 end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
155 end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
156
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
157 function archive:_get_idx(username, id, dates)
2938
f000ba14d531 mod_storage_xmlarchive: Add a debug message to try to track down why all indicies are read in some cases
Kim Alvefur <zash@zash.se>
parents: 2917
diff changeset
158 module:log("debug", "Looking for item with id %q", id);
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
159 dates = dates or self:dates(username) or empty;
3452
0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format
Kim Alvefur <zash@zash.se>
parents: 3435
diff changeset
160 local date = id:match("^%d%d%d%d%-%d%d%-%d%d");
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
161 local tick = get_nexttick();
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
162 for d = 1, #dates do
3452
0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format
Kim Alvefur <zash@zash.se>
parents: 3435
diff changeset
163 if not date or date == dates[d] then
2917
cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging
Kim Alvefur <zash@zash.se>
parents: 2815
diff changeset
164 module:log("debug", "Loading index for %s", dates[d]);
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
165 local items = dm.list_load(username .. "@" .. dates[d], self.host, self.store) or empty;
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
166 for i = 1, #items do
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
167 if items[i].id == id then
3462
d73ed7975d82 mod_storage_xmlarchive: Add some debug logging
Kim Alvefur <zash@zash.se>
parents: 3453
diff changeset
168 module:log("debug", "Found item!");
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
169 return d, i, items;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
170 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
171 end
3453
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive
Kim Alvefur <zash@zash.se>
parents: 3452
diff changeset
172 if date then
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive
Kim Alvefur <zash@zash.se>
parents: 3452
diff changeset
173 return; -- Assuming no duplicates
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive
Kim Alvefur <zash@zash.se>
parents: 3452
diff changeset
174 end
3452
0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format
Kim Alvefur <zash@zash.se>
parents: 3435
diff changeset
175 elseif date and date < dates[d] then
3462
d73ed7975d82 mod_storage_xmlarchive: Add some debug logging
Kim Alvefur <zash@zash.se>
parents: 3453
diff changeset
176 module:log("debug", "Skipping remaining dates after %s", date);
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
177 return; -- List is assumed to be sorted
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
178 end
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
179
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
180 -- insert pauses to allow other processing
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
181 if d % 14 == 0 then tick(); end
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
182 end
3462
d73ed7975d82 mod_storage_xmlarchive: Add some debug logging
Kim Alvefur <zash@zash.se>
parents: 3453
diff changeset
183 module:log("debug", "Item not found");
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
184 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
185
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
186 function archive:find(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
187 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
188 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
189
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
190 local result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
191 local function cb(_, stanza)
2514
d47a7e6e9adc mod_storage_xmlarchive: Raise error instead of warning if there's ever more than one stanza in a chunk (this indicates some kind of corruption)
Kim Alvefur <zash@zash.se>
parents: 2420
diff changeset
192 assert(not result, "Multiple items in chunk");
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
193 result = stanza;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
194 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
195
2602
324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session'
Kim Alvefur <zash@zash.se>
parents: 2601
diff changeset
196 local stream_session = { notopen = true };
2601
ba5757dc883d mod_storage_xmlarchive: Move XML callback table into a variable to break long line [luacheck]
Kim Alvefur <zash@zash.se>
parents: 2600
diff changeset
197 local stream_callbacks = { handlestanza = cb, stream_ns = "jabber:client", default_ns = "jabber:client" };
2602
324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session'
Kim Alvefur <zash@zash.se>
parents: 2601
diff changeset
198 local stream = new_stream(stream_session, stream_callbacks);
2675
101a2a0b8b33 mod_storage_xmlarchive: Pass username to dates
Kim Alvefur <zash@zash.se>
parents: 2663
diff changeset
199 local dates = self:dates(username) or empty;
1727
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
200 local function reset_stream()
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
201 stream:reset();
2602
324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session'
Kim Alvefur <zash@zash.se>
parents: 2601
diff changeset
202 stream_session.notopen = true;
1727
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
203 stream:feed(st.stanza("stream", { xmlns = "jabber:client" }):top_tag());
2602
324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session'
Kim Alvefur <zash@zash.se>
parents: 2601
diff changeset
204 stream_session.notopen = nil;
1727
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
205 end
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
206 reset_stream();
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
207
4649
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
208 local filtered_ids = false;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
209 local filtered_dates = false;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
210 if query.ids then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
211 filtered_ids = {};
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
212 filtered_dates = {};
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
213 for _, id in ipairs(query.ids) do
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
214 filtered_ids[id] = true;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
215 if filtered_dates then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
216 local date = id:match("^%d%d%d%d%-%d%d%-%d%d");
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
217 if date then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
218 filtered_dates[date] = true;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
219 else
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
220 -- if any id diverges from the standard then the item could be from any date
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
221 filtered_dates = nil;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
222 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
223 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
224 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
225 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
226
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
227 if filtered_dates then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
228 for i = #dates, 1, -1 do
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
229 if not filtered_dates[ dates[i] ] then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
230 table.remove(dates, i);
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
231 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
232 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
233 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
234
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
235 local limit = query.limit;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
236 local start_day, step, last_day = 1, 1, #dates;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
237 local count = 0;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
238 local rev = query.reverse;
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
239 if query.start then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
240 local d = dt.date(query.start);
3587
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
241 for i = start_day, last_day, step do
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
242 if dates[i] < d then
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
243 start_day = i + 1;
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
244 elseif dates[i] >= d then
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
245 start_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
246 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
247 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
248 end
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
249 if query["end"] then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
250 local d = dt.date(query["end"]);
3587
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
251 for i = last_day, start_day, -step do
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
252 if dates[i] > d then
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
253 last_day = i - 1;
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
254 elseif dates[i] <= d then
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
255 last_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
256 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
257 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
258 end
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
259 local items;
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
260 local first_item, last_item;
4648
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
261 local stop_at_id;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
262 if rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
263 start_day, step, last_day = last_day, -step, start_day;
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
264 if query.before then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
265 local before_day, before_item, items_ = self:_get_idx(username, query.before, dates);
3572
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
266 if not before_day then
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
267 return nil, "item-not-found";
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
268 elseif before_day <= start_day then
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
269 if before_item then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
270 first_item = before_item - 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
271 else
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
272 first_item = #items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
273 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
274 last_item = 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
275 start_day = before_day;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
276 items = items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
277 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
278 end
4648
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
279 if query.after then
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
280 stop_at_id = query.after;
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
281 end
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
282 elseif query.after then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
283 local after_day, after_item, items_ = self:_get_idx(username, query.after, dates);
3572
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
284 if not after_day then
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
285 return nil, "item-not-found";
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
286 elseif after_day >= start_day then
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
287 if after_item then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
288 first_item = after_item + 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
289 else
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
290 first_item = 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
291 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
292 last_item = #items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
293 start_day = after_day;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
294 items = items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
295 end
4648
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
296 if query.before then
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
297 stop_at_id = query.before;
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
298 end
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
299 elseif query.before then
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
300 stop_at_id = query.before;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
301 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
302
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
303 local date_open, xmlfile;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
304 local function read_xml(date, offset, length)
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
305 if xmlfile and date ~= date_open then
2917
cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging
Kim Alvefur <zash@zash.se>
parents: 2815
diff changeset
306 module:log("debug", "Closing XML file for %s", date_open);
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
307 xmlfile:close();
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
308 xmlfile = nil;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
309 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
310 if not xmlfile then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
311 date_open = date;
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
312 local filename = dm.getpath(username .. "@" .. date, self.host, self.store, "xml");
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
313 local ferr;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
314 xmlfile, ferr = io.open(filename);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
315 if not xmlfile then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
316 module:log("error", "Error: %s", ferr);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
317 return nil, ferr;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
318 end
2917
cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging
Kim Alvefur <zash@zash.se>
parents: 2815
diff changeset
319 module:log("debug", "Opened XML file %s", filename);
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
320 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
321 local pos, err = xmlfile:seek("set", offset);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
322 if pos ~= offset then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
323 return nil, err or "seek-failed";
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
324 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
325 return xmlfile:read(length);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
326 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
327
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
328 local tick = get_nexttick();
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
329
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
330 return function ()
2054
b7c528027762 mod_storage_xmlarchive: Only try to close xmlfile if it has been opened (fixes traceback with limit=0)
Kim Alvefur <zash@zash.se>
parents: 2041
diff changeset
331 if limit and count >= limit then if xmlfile then xmlfile:close() end return; end
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
332 for d = start_day, last_day, step do
2633
1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local
Kim Alvefur <zash@zash.se>
parents: 2632
diff changeset
333 local date = dates[d];
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
334 if not items then
2917
cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging
Kim Alvefur <zash@zash.se>
parents: 2815
diff changeset
335 module:log("debug", "Loading index for %s", date);
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
336 start_day = d;
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
337 items = dm.list_load(username .. "@" .. date, self.host, self.store) or empty;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
338 if not rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
339 first_item, last_item = 1, #items;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
340 else
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
341 first_item, last_item = #items, 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
342 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
343 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
344
1741
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop
Kim Alvefur <zash@zash.se>
parents: 1740
diff changeset
345 local q_with, q_start, q_end = query.with, query.start, query["end"];
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
346 for i = first_item, last_item, step do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
347 local item = items[i];
2396
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields
Kim Alvefur <zash@zash.se>
parents: 2290
diff changeset
348 if not item then
2633
1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local
Kim Alvefur <zash@zash.se>
parents: 2632
diff changeset
349 module:log("warn", "data[%q][%d] is nil", date, i);
2396
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields
Kim Alvefur <zash@zash.se>
parents: 2290
diff changeset
350 break;
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields
Kim Alvefur <zash@zash.se>
parents: 2290
diff changeset
351 end
2634
71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability
Kim Alvefur <zash@zash.se>
parents: 2633
diff changeset
352
1741
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop
Kim Alvefur <zash@zash.se>
parents: 1740
diff changeset
353 local i_when, i_with = item.when, item.with;
2634
71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability
Kim Alvefur <zash@zash.se>
parents: 2633
diff changeset
354
1742
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
355 if type(i_when) == "string" then
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
356 i_when = dt.parse(i_when);
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
357 end
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
358 if type(i_when) ~= "number" then
2633
1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local
Kim Alvefur <zash@zash.se>
parents: 2632
diff changeset
359 module:log("warn", "data[%q][%d].when is invalid", date, i);
1742
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
360 break;
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
361 end
2634
71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability
Kim Alvefur <zash@zash.se>
parents: 2633
diff changeset
362
4648
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
363 if stop_at_id and stop_at_id == item.id then
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
364 if xmlfile then xmlfile:close(); end
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
365 return;
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
366 end
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
367
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
368 if (not q_with or i_with == q_with)
1741
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop
Kim Alvefur <zash@zash.se>
parents: 1740
diff changeset
369 and (not q_start or i_when >= q_start)
4649
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
370 and (not q_end or i_when <= q_end)
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
371 and (not filtered_ids or filtered_ids[item.id]) then
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
372 count = count + 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
373 first_item = i + step;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
374
2633
1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local
Kim Alvefur <zash@zash.se>
parents: 2632
diff changeset
375 local data = read_xml(date, item.offset, item.length);
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
376 if not data then return end
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
377 local ok, err = stream:feed(data);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
378 if not ok then
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
379 module:log("warn", "Parse error in %s@%s/%s/%q[%d]: %s", username, self.host, self.store, i, err);
1727
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
380 reset_stream();
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
381 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
382 if result then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
383 local stanza = result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
384 result = nil;
1752
3f3689a16133 mod_storage_xmlarchive: Return 'when' as number
Kim Alvefur <zash@zash.se>
parents: 1742
diff changeset
385 return item.id, stanza, i_when, i_with;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
386 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
387 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
388 end
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
389 items = nil;
2397
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
390 if xmlfile then
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
391 xmlfile:close();
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
392 xmlfile = nil;
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
393 end
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
394 -- If we're running through a lot of day-files then lets allow for other processing between each day
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
395 tick();
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
396 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
397 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
398 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
399
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
400 function archive:delete(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
401 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
402 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
403 if query.with or query.start or query.after then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
404 return nil, "not-implemented"; -- Only trimming the oldest messages
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
405 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
406 local before = query.before or query["end"] or "9999-12-31";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
407 if type(before) == "number" then before = dt.date(before); else before = before:sub(1, 10); end
2663
6e8c3fc48237 mod_storage_xmlarchive: Use the dates method in more places
Kim Alvefur <zash@zash.se>
parents: 2634
diff changeset
408 local dates, err = self:dates(username);
2040
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
409 if not dates or next(dates) == nil then
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
410 if not err then return true end -- already empty
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
411 return dates, err;
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
412 end
2041
7c61ab512d0b mod_storage_xmlarchive: Return earlier if attempting to delete messages older than the oldest existing
Kim Alvefur <zash@zash.se>
parents: 2040
diff changeset
413 if dates[1] > before then return true; end -- Nothing to delete
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
414 local remaining_dates = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
415 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
416 if dates[d] >= before then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
417 table.insert(remaining_dates, dates[d]);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
418 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
419 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
420 table.sort(remaining_dates);
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
421 local ok, err = dm.list_store(username, self.host, self.store, remaining_dates);
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
422 if not ok then return ok, err; end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
423 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
424 if dates[d] < before then
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
425 os.remove(dm.getpath(username .. "@" .. dates[d], self.host, self.store, "list"));
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
426 os.remove(dm.getpath(username .. "@" .. dates[d], self.host, self.store, "xml"));
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
427 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
428 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
429 return true;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
430 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
431
1831
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages
Kim Alvefur <zash@zash.se>
parents: 1819
diff changeset
432 function archive:dates(username)
2917
cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging
Kim Alvefur <zash@zash.se>
parents: 2815
diff changeset
433 module:log("debug", "Loading root index for %s", username);
2814
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives
Kim Alvefur <zash@zash.se>
parents: 2813
diff changeset
434 local dates, err = dm.list_load(username, self.host, self.store);
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives
Kim Alvefur <zash@zash.se>
parents: 2813
diff changeset
435 if not dates then return dates, err; end
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives
Kim Alvefur <zash@zash.se>
parents: 2813
diff changeset
436 assert(type(dates[1]) == "string" and type(dates[#dates]) == "string",
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives
Kim Alvefur <zash@zash.se>
parents: 2813
diff changeset
437 "Archive does not appear to be in xmlarchive format");
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives
Kim Alvefur <zash@zash.se>
parents: 2813
diff changeset
438 return dates;
1831
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages
Kim Alvefur <zash@zash.se>
parents: 1819
diff changeset
439 end
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages
Kim Alvefur <zash@zash.se>
parents: 1819
diff changeset
440
3585
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API
Kim Alvefur <zash@zash.se>
parents: 3572
diff changeset
441 function archive:users()
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API
Kim Alvefur <zash@zash.se>
parents: 3572
diff changeset
442 return dm.users(module.host, self.store, "list");
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API
Kim Alvefur <zash@zash.se>
parents: 3572
diff changeset
443 end
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API
Kim Alvefur <zash@zash.se>
parents: 3572
diff changeset
444
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
445 local provider = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
446 function provider:open(store, typ)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
447 if typ ~= "archive" then return nil, "unsupported-store"; end
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
448 return setmetatable({ host = module.host, store = store }, archive_mt);
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
449 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
450
1819
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
451 function provider:purge(username)
2676
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
452 local encoded_username = dm.path_encode((username or "@") .. "@");
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
453 local basepath = prosody.paths.data .. "/" .. dm.path_encode(module.host);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
454 for store in lfs.dir(basepath) do
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
455 store = basepath .. "/" .. dm.path_encode(store);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
456 if lfs.attributes(store, "mode") == "directory" then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
457 for file in lfs.dir(store) do
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
458 if file:sub(1, #encoded_username) == encoded_username then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
459 if file:sub(-4) == ".xml" or file:sub(-5) == ".list" then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
460 os.remove(store .. "/" .. file);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
461 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
462 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
463 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
464 return true;
1819
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
465 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
466 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
467 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
468
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
469 module:provides("storage", provider);
2815
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
470
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
471
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
472 function module.command(arg)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
473 local jid = require "util.jid";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
474 if arg[1] == "convert" and (arg[2] == "to" or arg[2] == "from") and arg[4] then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
475 local convert;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
476 if arg[2] == "to" then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
477 function convert(user, host, store)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
478 local dates, err = archive.dates({ host = host, store = store }, user);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
479 if not dates then assert(not err, err); return end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
480 assert(dm.list_store(user, host, store, nil));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
481 for _, date in ipairs(dates) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
482 print(date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
483 local items = assert(dm.list_load(user .. "@" .. date, host, store));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
484 local xmlfile = assert(io.open(dm.getpath(user .. "@" .. date, host, store, "xml")));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
485 for _, item in ipairs(items) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
486 assert(xmlfile:seek("set", item.offset));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
487 local data = assert(xmlfile:read(item.length));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
488 assert(#data == item.length, "short read");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
489 data = assert(xml.parse(data));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
490 data = st.preserialize(data);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
491 data.key = item.id;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
492 data.with = item.with;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
493 data.when = tonumber(item.when) or dt.parse(item.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
494 data.attr.stamp = item.when;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
495 assert(dm.list_append(user, host, store, data));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
496 end
3491
4e9d4b07e3e9 mod_storage_xmlarchive: Remove per day files during migration
Kim Alvefur <zash@zash.se>
parents: 3462
diff changeset
497 assert(os.remove(dm.getpath(user .. "@" .. date, host, store, "list")));
4e9d4b07e3e9 mod_storage_xmlarchive: Remove per day files during migration
Kim Alvefur <zash@zash.se>
parents: 3462
diff changeset
498 assert(os.remove(dm.getpath(user .. "@" .. date, host, store, "xml")));
2815
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
499 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
500 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
501 else -- convert from internal
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
502 function convert(user, host, store)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
503 local items, err = dm.list_load(user, host, store);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
504 if not items then assert(not err, err); return end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
505 local dates = {};
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
506 local dayitems, date, xmlfile;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
507 for _, item in ipairs(items) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
508 local meta = {
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
509 id = item.key;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
510 with = item.with;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
511 when = item.when or dt.parse(item.attr.stamp);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
512 };
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
513 local current_date = dt.date(meta.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
514 if current_date ~= date then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
515 if xmlfile then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
516 assert(xmlfile:close());
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
517 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
518 if dayitems then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
519 assert(dm.list_store(user .. "@" .. date, host, store, dayitems));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
520 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
521 print(current_date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
522 dayitems = {};
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
523 date = current_date;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
524 table.insert(dates, date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
525 xmlfile = assert(io.open(dm.getpath(user .. "@" .. date, host, store, "xml"), "w"));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
526 end
4671
98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion
Kim Alvefur <zash@zash.se>
parents: 4649
diff changeset
527 -- The property 'stamp_legacy' originally came from mod_offline and
98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion
Kim Alvefur <zash@zash.se>
parents: 4649
diff changeset
528 -- was inherited into the mod_storage_internal archive format for
98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion
Kim Alvefur <zash@zash.se>
parents: 4649
diff changeset
529 -- compat. It should not be needed since Prosody 0.10.0 but could
98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion
Kim Alvefur <zash@zash.se>
parents: 4649
diff changeset
530 -- exist in older data.
2815
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
531 item.attr.stamp, item.attr.stamp_legacy = nil, nil;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
532 local stanza = tostring(st.deserialize(item)) .. "\n";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
533 meta.offset, meta.length = xmlfile:seek(), #stanza;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
534 assert(xmlfile:write(stanza));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
535 table.insert(dayitems, meta);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
536 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
537 assert(xmlfile:close());
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
538 assert(dm.list_store(user .. "@" .. date, host, store, dayitems));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
539 assert(dm.list_store(user, host, store, dates));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
540 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
541 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
542
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
543 local store = arg[4];
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
544 if arg[3] == "internal" then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
545 for i = 5, #arg do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
546 local user, host = jid.prepped_split(arg[i]);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
547 if not user then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
548 print(string.format("Argument #%d (%q) is an invalid JID, aborting", i, arg[i]));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
549 os.exit(1);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
550 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
551 convert(user, host, store);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
552 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
553 print("Done");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
554 return 0;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
555 else
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
556 print("Currently only conversion to/from mod_storage_internal is supported");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
557 print("Check out https://modules.prosody.im/mod_migrate");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
558 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
559 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
560 print("prosodyctl mod_storage_xmlarchive convert (from|to) internal (archive|archive2|muc_log) user@host");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
561 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
562