Mercurial > prosody-modules

annotate mod_storage_xmlarchive/mod_storage_xmlarchive.lua @ 5820:742142f9771e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mod_groups_internal: Fix traceback when room doesn't exist I'm not sure if it should even be included if it doesn't exist, but I'm not currently sure how this situation occurs, so I'm implementing the lightest possible fix for now.
author Matthew Wild <mwild1@gmail.com>
date Thu, 11 Jan 2024 15:53:18 +0000
parents 645de410dbca
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1764
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
1 -- mod_storage_xmlarchive
4647
b91e40472d68 mod_storage_xmlarchive: Fix return of numeric 'when' from key-value API
Kim Alvefur <zash@zash.se>
parents: 4646
diff changeset
2 -- Copyright (C) 2015-2021 Kim Alvefur
1764
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
3 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
4 -- This file is MIT/X11 licensed.
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
5 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
6 -- luacheck: ignore unused self
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
7
2290
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
8 local lfs = require "lfs";
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
9 local dm = require "core.storagemanager".olddm;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
10 local hmac_sha256 = require"util.hashes".hmac_sha256;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
11 local st = require"util.stanza";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
12 local dt = require"util.datetime";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
13 local new_stream = require "util.xmppstream".new;
3898
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
14 local xml = require "util.xml";
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
15 local async = require "util.async";
5729
79ba1a1a75cc mod_storage_xmlarchive: Fix "user" iteration API
Kim Alvefur <zash@zash.se>
parents: 4671
diff changeset
16 local it = require "util.iterators";
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
17 local empty = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
18
2289
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
19 if not dm.append_raw then
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
20 module:require"datamanager_append_raw";
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
21 end
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
22
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
23 local archive = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
24 local archive_mt = { __index = archive };
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
25
4646
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm)
Kim Alvefur <zash@zash.se>
parents: 4551
diff changeset
26 archive.caps = {
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm)
Kim Alvefur <zash@zash.se>
parents: 4551
diff changeset
27 total = false,
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm)
Kim Alvefur <zash@zash.se>
parents: 4551
diff changeset
28 quota = nil,
4648
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
29 full_id_range = true;
4649
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
30 ids = true;
4646
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm)
Kim Alvefur <zash@zash.se>
parents: 4551
diff changeset
31 };
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm)
Kim Alvefur <zash@zash.se>
parents: 4551
diff changeset
32
2420
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available
Kim Alvefur <zash@zash.se>
parents: 2405
diff changeset
33 local is_stanza = st.is_stanza or function (s)
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available
Kim Alvefur <zash@zash.se>
parents: 2405
diff changeset
34 return getmetatable(s) == st.stanza_mt;
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available
Kim Alvefur <zash@zash.se>
parents: 2405
diff changeset
35 end
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available
Kim Alvefur <zash@zash.se>
parents: 2405
diff changeset
36
5742
645de410dbca mod_storage_xmlarchive: Support using requested archive-id
Kim Alvefur <zash@zash.se>
parents: 5741
diff changeset
37 function archive:append(username, id, data, when, with)
2420
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available
Kim Alvefur <zash@zash.se>
parents: 2405
diff changeset
38 if not is_stanza(data) then
1739
940a4ab75cec mod_storage_xmlarchive: This module only supports storage of stanzas, log a warning about attempts to do otherwise
Kim Alvefur <zash@zash.se>
parents: 1738
diff changeset
39 module:log("error", "Attempt to store non-stanza object, traceback: %s", debug.traceback());
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
40 return nil, "unsupported-datatype";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
41 end
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
42
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
43 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
44 data = tostring(data) .. "\n";
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
45
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
46 local day = dt.date(when);
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
47 local ok, err = dm.append_raw(username.."@"..day, self.host, self.store, "xml", data);
1969
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
48 if not ok then
3755
bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't
Kim Alvefur <zash@zash.se>
parents: 3587
diff changeset
49 -- append_raw, unlike list_append, does not log anything on failure atm, so we do so here
bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't
Kim Alvefur <zash@zash.se>
parents: 3587
diff changeset
50 module:log("error", "Unable to write to %s storage ('%s') for user: %s@%s",
bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't
Kim Alvefur <zash@zash.se>
parents: 3587
diff changeset
51 self.store, err, username, module.host)
1969
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
52 return nil, err;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
53 end
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
54
2290
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
55 -- If the day-file is missing then we need to add it to the list of days
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
56 local first_of_day = not lfs.attributes(dm.getpath(username .. "@" .. day, self.host, self.store, "list"));
2290
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
57
2595
307ddebb72e1 mod_storage_xmlarchive: Assume offset to be zero if not included (thanks pep.)
Kim Alvefur <zash@zash.se>
parents: 2514
diff changeset
58 local offset = ok and err or 0;
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
59
5742
645de410dbca mod_storage_xmlarchive: Support using requested archive-id
Kim Alvefur <zash@zash.se>
parents: 5741
diff changeset
60 id = id or day .. "-" .. hmac_sha256(username.."@"..day.."+"..offset, data, true):sub(-16);
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
61 ok, err = dm.list_append(username.."@"..day, self.host, self.store,
2600
9ec8289f6bb2 mod_storage_xmlarchive: Break long line [luacheck]
Kim Alvefur <zash@zash.se>
parents: 2595
diff changeset
62 { id = id, when = dt.datetime(when), with = with, offset = offset, length = #data });
2290
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
63 if ok and first_of_day then
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
64 ok, err = dm.list_append(username, self.host, self.store, day);
1969
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
65 end
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
66 if not ok then
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
67 return nil, err;
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
68 end
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
69 return id;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
70 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
71
3898
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
72 function archive:get(username, id)
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
73 local dates = self:dates(username) or empty;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
74 local day_idx, item_idx, items = self:_get_idx(username, id, dates);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
75 if not day_idx then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
76 return nil, "item-not-found";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
77 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
78 module:log("debug", ":_get_idx(%q, %q) --> %q, %q", username, id, dates[day_idx], items[item_idx]);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
79 local day = dates[day_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
80 local item = items[item_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
81 module:log("debug", "item = %q", item);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
82 local filename = dm.getpath(username.."@"..day, self.host, self.store, "xml");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
83 local xmlfile, ferr = io.open(filename, "r");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
84 if not xmlfile then return nil, ferr; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
85 local p,err = xmlfile:seek("set", item.offset);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
86 if p ~= item.offset or err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
87 local data = xmlfile:read(item.length);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
88 local parsed, perr = xml.parse(data);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
89 if not parsed then return nil, perr; end
4647
b91e40472d68 mod_storage_xmlarchive: Fix return of numeric 'when' from key-value API
Kim Alvefur <zash@zash.se>
parents: 4646
diff changeset
90 return parsed, tonumber(item.when) or dt.parse(item.when), item.with;
3898
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
91 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
92
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
93 local overwrite = module:get_option("xmlarchive_overwrite", false);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
94
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
95 function archive:set(username, id, data, new_when, new_with)
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
96 if not is_stanza(data) then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
97 module:log("error", "Attempt to store non-stanza object, traceback: %s", debug.traceback());
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
98 return nil, "unsupported-datatype";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
99 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
100
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
101 username = username or "@";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
102 data = tostring(data) .. "\n";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
103
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
104 local dates = self:dates(username) or empty;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
105 local day_idx, item_idx, items = self:_get_idx(username, id, dates);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
106 if not day_idx then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
107 return nil, "item-not-found";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
108 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
109 local day = dates[day_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
110 local item = items[item_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
111
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
112 local filename = dm.getpath(username.."@"..day, self.host, self.store, "xml");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
113
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
114 local replaced, err = dm.append_raw(username.."@"..day, self.host, self.store, "xml", data);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
115 if not replaced then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
116 local new_offset = err;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
117
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
118 -- default yes or no?
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
119 if overwrite then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
120 local xmlfile, ferr = io.open(filename, "r+");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
121 if not xmlfile then return nil, ferr; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
122 local p,err = xmlfile:seek("set", item.offset);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
123 if p ~= item.offset or err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
124 local _,err = xmlfile:write((" "):rep(item.length));
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
125 if err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
126 local _,err = xmlfile:close();
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
127 if err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
128 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
129
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
130 items[item_idx] = {
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
131 id = id,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
132 when = new_when or item.when,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
133 with = new_with or item.with,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
134 offset = new_offset,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
135 length = #data,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
136 replaces = item,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
137 };
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
138 local ok, err = dm.list_store(username.."@"..day, self.host, self.store, items);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
139 return ok, err;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
140 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores
Kim Alvefur <zash@zash.se>
parents: 3755
diff changeset
141
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
142 local function get_nexttick()
4551
b92147edd172 mod_storage_xmlarchive: Workaround for #1646 (util.async bug with Lua 5.1)
Kim Alvefur <zash@zash.se>
parents: 4538
diff changeset
143 -- COMPAT util.async under Lua 5.1 runs into problems trying to yield across
b92147edd172 mod_storage_xmlarchive: Workaround for #1646 (util.async bug with Lua 5.1)
Kim Alvefur <zash@zash.se>
parents: 4538
diff changeset
144 -- pcall barriers. Workaround for #1646
b92147edd172 mod_storage_xmlarchive: Workaround for #1646 (util.async bug with Lua 5.1)
Kim Alvefur <zash@zash.se>
parents: 4538
diff changeset
145 if _VERSION ~= "Lua 5.1" and async.ready() then
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
146 return function ()
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
147 -- slow down
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
148 local wait, done = async.waiter();
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
149 module:add_timer(0, done);
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
150 wait();
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
151 end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
152 else
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
153 -- no async, no-op
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
154 return function () end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
155 end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
156 end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
157
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
158 function archive:_get_idx(username, id, dates)
2938
f000ba14d531 mod_storage_xmlarchive: Add a debug message to try to track down why all indicies are read in some cases
Kim Alvefur <zash@zash.se>
parents: 2917
diff changeset
159 module:log("debug", "Looking for item with id %q", id);
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
160 dates = dates or self:dates(username) or empty;
3452
0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format
Kim Alvefur <zash@zash.se>
parents: 3435
diff changeset
161 local date = id:match("^%d%d%d%d%-%d%d%-%d%d");
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
162 local tick = get_nexttick();
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
163 for d = 1, #dates do
3452
0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format
Kim Alvefur <zash@zash.se>
parents: 3435
diff changeset
164 if not date or date == dates[d] then
2917
cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging
Kim Alvefur <zash@zash.se>
parents: 2815
diff changeset
165 module:log("debug", "Loading index for %s", dates[d]);
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
166 local items = dm.list_load(username .. "@" .. dates[d], self.host, self.store) or empty;
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
167 for i = 1, #items do
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
168 if items[i].id == id then
3462
d73ed7975d82 mod_storage_xmlarchive: Add some debug logging
Kim Alvefur <zash@zash.se>
parents: 3453
diff changeset
169 module:log("debug", "Found item!");
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
170 return d, i, items;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
171 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
172 end
3453
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive
Kim Alvefur <zash@zash.se>
parents: 3452
diff changeset
173 if date then
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive
Kim Alvefur <zash@zash.se>
parents: 3452
diff changeset
174 return; -- Assuming no duplicates
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive
Kim Alvefur <zash@zash.se>
parents: 3452
diff changeset
175 end
3452
0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format
Kim Alvefur <zash@zash.se>
parents: 3435
diff changeset
176 elseif date and date < dates[d] then
3462
d73ed7975d82 mod_storage_xmlarchive: Add some debug logging
Kim Alvefur <zash@zash.se>
parents: 3453
diff changeset
177 module:log("debug", "Skipping remaining dates after %s", date);
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
178 return; -- List is assumed to be sorted
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
179 end
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
180
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
181 -- insert pauses to allow other processing
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
182 if d % 14 == 0 then tick(); end
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
183 end
3462
d73ed7975d82 mod_storage_xmlarchive: Add some debug logging
Kim Alvefur <zash@zash.se>
parents: 3453
diff changeset
184 module:log("debug", "Item not found");
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
185 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
186
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
187 function archive:find(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
188 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
189 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
190
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
191 local result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
192 local function cb(_, stanza)
2514
d47a7e6e9adc mod_storage_xmlarchive: Raise error instead of warning if there's ever more than one stanza in a chunk (this indicates some kind of corruption)
Kim Alvefur <zash@zash.se>
parents: 2420
diff changeset
193 assert(not result, "Multiple items in chunk");
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
194 result = stanza;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
195 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
196
2602
324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session'
Kim Alvefur <zash@zash.se>
parents: 2601
diff changeset
197 local stream_session = { notopen = true };
2601
ba5757dc883d mod_storage_xmlarchive: Move XML callback table into a variable to break long line [luacheck]
Kim Alvefur <zash@zash.se>
parents: 2600
diff changeset
198 local stream_callbacks = { handlestanza = cb, stream_ns = "jabber:client", default_ns = "jabber:client" };
2602
324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session'
Kim Alvefur <zash@zash.se>
parents: 2601
diff changeset
199 local stream = new_stream(stream_session, stream_callbacks);
2675
101a2a0b8b33 mod_storage_xmlarchive: Pass username to dates
Kim Alvefur <zash@zash.se>
parents: 2663
diff changeset
200 local dates = self:dates(username) or empty;
1727
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
201 local function reset_stream()
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
202 stream:reset();
2602
324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session'
Kim Alvefur <zash@zash.se>
parents: 2601
diff changeset
203 stream_session.notopen = true;
1727
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
204 stream:feed(st.stanza("stream", { xmlns = "jabber:client" }):top_tag());
2602
324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session'
Kim Alvefur <zash@zash.se>
parents: 2601
diff changeset
205 stream_session.notopen = nil;
1727
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
206 end
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
207 reset_stream();
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
208
4649
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
209 local filtered_ids = false;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
210 local filtered_dates = false;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
211 if query.ids then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
212 filtered_ids = {};
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
213 filtered_dates = {};
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
214 for _, id in ipairs(query.ids) do
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
215 filtered_ids[id] = true;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
216 if filtered_dates then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
217 local date = id:match("^%d%d%d%d%-%d%d%-%d%d");
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
218 if date then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
219 filtered_dates[date] = true;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
220 else
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
221 -- if any id diverges from the standard then the item could be from any date
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
222 filtered_dates = nil;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
223 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
224 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
225 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
226 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
227
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
228 if filtered_dates then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
229 for i = #dates, 1, -1 do
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
230 if not filtered_dates[ dates[i] ] then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
231 table.remove(dates, i);
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
232 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
233 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
234 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
235
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
236 local limit = query.limit;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
237 local start_day, step, last_day = 1, 1, #dates;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
238 local count = 0;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
239 local rev = query.reverse;
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
240 if query.start then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
241 local d = dt.date(query.start);
3587
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
242 for i = start_day, last_day, step do
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
243 if dates[i] < d then
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
244 start_day = i + 1;
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
245 elseif dates[i] >= d then
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
246 start_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
247 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
248 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
249 end
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
250 if query["end"] then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
251 local d = dt.date(query["end"]);
3587
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
252 for i = last_day, start_day, -step do
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
253 if dates[i] > d then
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
254 last_day = i - 1;
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check
Kim Alvefur <zash@zash.se>
parents: 3585
diff changeset
255 elseif dates[i] <= d then
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
256 last_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
257 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
258 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
259 end
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
260 local items;
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
261 local first_item, last_item;
4648
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
262 local stop_at_id;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
263 if rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
264 start_day, step, last_day = last_day, -step, start_day;
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
265 if query.before then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
266 local before_day, before_item, items_ = self:_get_idx(username, query.before, dates);
3572
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
267 if not before_day then
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
268 return nil, "item-not-found";
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
269 elseif before_day <= start_day then
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
270 if before_item then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
271 first_item = before_item - 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
272 else
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
273 first_item = #items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
274 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
275 last_item = 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
276 start_day = before_day;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
277 items = items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
278 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
279 end
4648
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
280 if query.after then
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
281 stop_at_id = query.after;
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
282 end
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
283 elseif query.after then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
284 local after_day, after_item, items_ = self:_get_idx(username, query.after, dates);
3572
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
285 if not after_day then
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
286 return nil, "item-not-found";
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325)
Kim Alvefur <zash@zash.se>
parents: 3571
diff changeset
287 elseif after_day >= start_day then
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
288 if after_item then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
289 first_item = after_item + 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
290 else
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
291 first_item = 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
292 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
293 last_item = #items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
294 start_day = after_day;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
295 items = items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
296 end
4648
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
297 if query.before then
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
298 stop_at_id = query.before;
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
299 end
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
300 elseif query.before then
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
301 stop_at_id = query.before;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
302 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
303
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
304 local date_open, xmlfile;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
305 local function read_xml(date, offset, length)
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
306 if xmlfile and date ~= date_open then
2917
cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging
Kim Alvefur <zash@zash.se>
parents: 2815
diff changeset
307 module:log("debug", "Closing XML file for %s", date_open);
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
308 xmlfile:close();
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
309 xmlfile = nil;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
310 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
311 if not xmlfile then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
312 date_open = date;
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
313 local filename = dm.getpath(username .. "@" .. date, self.host, self.store, "xml");
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
314 local ferr;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
315 xmlfile, ferr = io.open(filename);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
316 if not xmlfile then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
317 module:log("error", "Error: %s", ferr);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
318 return nil, ferr;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
319 end
2917
cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging
Kim Alvefur <zash@zash.se>
parents: 2815
diff changeset
320 module:log("debug", "Opened XML file %s", filename);
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
321 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
322 local pos, err = xmlfile:seek("set", offset);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
323 if pos ~= offset then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
324 return nil, err or "seek-failed";
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
325 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
326 return xmlfile:read(length);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
327 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
328
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
329 local tick = get_nexttick();
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
330
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
331 return function ()
2054
b7c528027762 mod_storage_xmlarchive: Only try to close xmlfile if it has been opened (fixes traceback with limit=0)
Kim Alvefur <zash@zash.se>
parents: 2041
diff changeset
332 if limit and count >= limit then if xmlfile then xmlfile:close() end return; end
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
333 for d = start_day, last_day, step do
2633
1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local
Kim Alvefur <zash@zash.se>
parents: 2632
diff changeset
334 local date = dates[d];
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
335 if not items then
2917
cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging
Kim Alvefur <zash@zash.se>
parents: 2815
diff changeset
336 module:log("debug", "Loading index for %s", date);
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
337 start_day = d;
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
338 items = dm.list_load(username .. "@" .. date, self.host, self.store) or empty;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
339 if not rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
340 first_item, last_item = 1, #items;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
341 else
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
342 first_item, last_item = #items, 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
343 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
344 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
345
1741
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop
Kim Alvefur <zash@zash.se>
parents: 1740
diff changeset
346 local q_with, q_start, q_end = query.with, query.start, query["end"];
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
347 for i = first_item, last_item, step do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
348 local item = items[i];
2396
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields
Kim Alvefur <zash@zash.se>
parents: 2290
diff changeset
349 if not item then
2633
1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local
Kim Alvefur <zash@zash.se>
parents: 2632
diff changeset
350 module:log("warn", "data[%q][%d] is nil", date, i);
2396
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields
Kim Alvefur <zash@zash.se>
parents: 2290
diff changeset
351 break;
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields
Kim Alvefur <zash@zash.se>
parents: 2290
diff changeset
352 end
2634
71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability
Kim Alvefur <zash@zash.se>
parents: 2633
diff changeset
353
1741
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop
Kim Alvefur <zash@zash.se>
parents: 1740
diff changeset
354 local i_when, i_with = item.when, item.with;
2634
71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability
Kim Alvefur <zash@zash.se>
parents: 2633
diff changeset
355
1742
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
356 if type(i_when) == "string" then
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
357 i_when = dt.parse(i_when);
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
358 end
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
359 if type(i_when) ~= "number" then
2633
1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local
Kim Alvefur <zash@zash.se>
parents: 2632
diff changeset
360 module:log("warn", "data[%q][%d].when is invalid", date, i);
1742
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
361 break;
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
362 end
2634
71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability
Kim Alvefur <zash@zash.se>
parents: 2633
diff changeset
363
4648
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
364 if stop_at_id and stop_at_id == item.id then
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
365 if xmlfile then xmlfile:close(); end
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
366 return;
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
367 end
37ca6109077f mod_storage_xmlarchive: Support full id range queries
Kim Alvefur <zash@zash.se>
parents: 4647
diff changeset
368
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
369 if (not q_with or i_with == q_with)
1741
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop
Kim Alvefur <zash@zash.se>
parents: 1740
diff changeset
370 and (not q_start or i_when >= q_start)
4649
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
371 and (not q_end or i_when <= q_end)
62d41447615d mod_storage_xmlarchive: Support query by list of IDs
Kim Alvefur <zash@zash.se>
parents: 4648
diff changeset
372 and (not filtered_ids or filtered_ids[item.id]) then
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
373 count = count + 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
374 first_item = i + step;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
375
2633
1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local
Kim Alvefur <zash@zash.se>
parents: 2632
diff changeset
376 local data = read_xml(date, item.offset, item.length);
2632
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function
Kim Alvefur <zash@zash.se>
parents: 2602
diff changeset
377 if not data then return end
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
378 local ok, err = stream:feed(data);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
379 if not ok then
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
380 module:log("warn", "Parse error in %s@%s/%s/%q[%d]: %s", username, self.host, self.store, i, err);
1727
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
381 reset_stream();
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
382 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
383 if result then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
384 local stanza = result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
385 result = nil;
1752
3f3689a16133 mod_storage_xmlarchive: Return 'when' as number
Kim Alvefur <zash@zash.se>
parents: 1742
diff changeset
386 return item.id, stanza, i_when, i_with;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
387 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
388 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
389 end
2398
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ)
Kim Alvefur <zash@zash.se>
parents: 2397
diff changeset
390 items = nil;
2397
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
391 if xmlfile then
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
392 xmlfile:close();
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
393 xmlfile = nil;
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
394 end
4538
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
395 -- If we're running through a lot of day-files then lets allow for other processing between each day
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries
Kim Alvefur <zash@zash.se>
parents: 3899
diff changeset
396 tick();
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
397 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
398 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
399 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
400
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
401 function archive:delete(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
402 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
403 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
404 if query.with or query.start or query.after then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
405 return nil, "not-implemented"; -- Only trimming the oldest messages
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
406 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
407 local before = query.before or query["end"] or "9999-12-31";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
408 if type(before) == "number" then before = dt.date(before); else before = before:sub(1, 10); end
2663
6e8c3fc48237 mod_storage_xmlarchive: Use the dates method in more places
Kim Alvefur <zash@zash.se>
parents: 2634
diff changeset
409 local dates, err = self:dates(username);
2040
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
410 if not dates or next(dates) == nil then
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
411 if not err then return true end -- already empty
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
412 return dates, err;
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
413 end
2041
7c61ab512d0b mod_storage_xmlarchive: Return earlier if attempting to delete messages older than the oldest existing
Kim Alvefur <zash@zash.se>
parents: 2040
diff changeset
414 if dates[1] > before then return true; end -- Nothing to delete
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
415 local remaining_dates = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
416 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
417 if dates[d] >= before then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
418 table.insert(remaining_dates, dates[d]);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
419 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
420 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
421 table.sort(remaining_dates);
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
422 local ok, err = dm.list_store(username, self.host, self.store, remaining_dates);
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
423 if not ok then return ok, err; end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
424 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
425 if dates[d] < before then
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
426 os.remove(dm.getpath(username .. "@" .. dates[d], self.host, self.store, "list"));
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
427 os.remove(dm.getpath(username .. "@" .. dates[d], self.host, self.store, "xml"));
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
428 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
429 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
430 return true;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
431 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
432
1831
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages
Kim Alvefur <zash@zash.se>
parents: 1819
diff changeset
433 function archive:dates(username)
2917
cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging
Kim Alvefur <zash@zash.se>
parents: 2815
diff changeset
434 module:log("debug", "Loading root index for %s", username);
2814
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives
Kim Alvefur <zash@zash.se>
parents: 2813
diff changeset
435 local dates, err = dm.list_load(username, self.host, self.store);
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives
Kim Alvefur <zash@zash.se>
parents: 2813
diff changeset
436 if not dates then return dates, err; end
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives
Kim Alvefur <zash@zash.se>
parents: 2813
diff changeset
437 assert(type(dates[1]) == "string" and type(dates[#dates]) == "string",
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives
Kim Alvefur <zash@zash.se>
parents: 2813
diff changeset
438 "Archive does not appear to be in xmlarchive format");
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives
Kim Alvefur <zash@zash.se>
parents: 2813
diff changeset
439 return dates;
1831
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages
Kim Alvefur <zash@zash.se>
parents: 1819
diff changeset
440 end
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages
Kim Alvefur <zash@zash.se>
parents: 1819
diff changeset
441
5729
79ba1a1a75cc mod_storage_xmlarchive: Fix "user" iteration API
Kim Alvefur <zash@zash.se>
parents: 4671
diff changeset
442 -- filter out the 'user@yyyy-mm-dd' stores
79ba1a1a75cc mod_storage_xmlarchive: Fix "user" iteration API
Kim Alvefur <zash@zash.se>
parents: 4671
diff changeset
443 local function skip_at_date(item)
79ba1a1a75cc mod_storage_xmlarchive: Fix "user" iteration API
Kim Alvefur <zash@zash.se>
parents: 4671
diff changeset
444 return not item:find("@");
79ba1a1a75cc mod_storage_xmlarchive: Fix "user" iteration API
Kim Alvefur <zash@zash.se>
parents: 4671
diff changeset
445 end
79ba1a1a75cc mod_storage_xmlarchive: Fix "user" iteration API
Kim Alvefur <zash@zash.se>
parents: 4671
diff changeset
446
3585
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API
Kim Alvefur <zash@zash.se>
parents: 3572
diff changeset
447 function archive:users()
5740
100110d539d3 mod_storage_xmlarchive: Migrate all users/rooms if no JID argument given
Kim Alvefur <zash@zash.se>
parents: 5729
diff changeset
448 return it.filter(skip_at_date, dm.users(self.host, self.store, "list"));
3585
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API
Kim Alvefur <zash@zash.se>
parents: 3572
diff changeset
449 end
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API
Kim Alvefur <zash@zash.se>
parents: 3572
diff changeset
450
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
451 local provider = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
452 function provider:open(store, typ)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
453 if typ ~= "archive" then return nil, "unsupported-store"; end
2813
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained
Kim Alvefur <zash@zash.se>
parents: 2676
diff changeset
454 return setmetatable({ host = module.host, store = store }, archive_mt);
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
455 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
456
1819
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
457 function provider:purge(username)
2676
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
458 local encoded_username = dm.path_encode((username or "@") .. "@");
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
459 local basepath = prosody.paths.data .. "/" .. dm.path_encode(module.host);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
460 for store in lfs.dir(basepath) do
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
461 store = basepath .. "/" .. dm.path_encode(store);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
462 if lfs.attributes(store, "mode") == "directory" then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
463 for file in lfs.dir(store) do
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
464 if file:sub(1, #encoded_username) == encoded_username then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
465 if file:sub(-4) == ".xml" or file:sub(-5) == ".list" then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
466 os.remove(store .. "/" .. file);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
467 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
468 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
469 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725)
Kim Alvefur <zash@zash.se>
parents: 2675
diff changeset
470 return true;
1819
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
471 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
472 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
473 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
474
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
475 module:provides("storage", provider);
2815
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
476
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
477
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
478 function module.command(arg)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
479 local jid = require "util.jid";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
480 if arg[1] == "convert" and (arg[2] == "to" or arg[2] == "from") and arg[4] then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
481 local convert;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
482 if arg[2] == "to" then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
483 function convert(user, host, store)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
484 local dates, err = archive.dates({ host = host, store = store }, user);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
485 if not dates then assert(not err, err); return end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
486 assert(dm.list_store(user, host, store, nil));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
487 for _, date in ipairs(dates) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
488 print(date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
489 local items = assert(dm.list_load(user .. "@" .. date, host, store));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
490 local xmlfile = assert(io.open(dm.getpath(user .. "@" .. date, host, store, "xml")));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
491 for _, item in ipairs(items) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
492 assert(xmlfile:seek("set", item.offset));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
493 local data = assert(xmlfile:read(item.length));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
494 assert(#data == item.length, "short read");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
495 data = assert(xml.parse(data));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
496 data = st.preserialize(data);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
497 data.key = item.id;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
498 data.with = item.with;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
499 data.when = tonumber(item.when) or dt.parse(item.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
500 data.attr.stamp = item.when;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
501 assert(dm.list_append(user, host, store, data));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
502 end
3491
4e9d4b07e3e9 mod_storage_xmlarchive: Remove per day files during migration
Kim Alvefur <zash@zash.se>
parents: 3462
diff changeset
503 assert(os.remove(dm.getpath(user .. "@" .. date, host, store, "list")));
4e9d4b07e3e9 mod_storage_xmlarchive: Remove per day files during migration
Kim Alvefur <zash@zash.se>
parents: 3462
diff changeset
504 assert(os.remove(dm.getpath(user .. "@" .. date, host, store, "xml")));
2815
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
505 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
506 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
507 else -- convert from internal
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
508 function convert(user, host, store)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
509 local items, err = dm.list_load(user, host, store);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
510 if not items then assert(not err, err); return end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
511 local dates = {};
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
512 local dayitems, date, xmlfile;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
513 for _, item in ipairs(items) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
514 local meta = {
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
515 id = item.key;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
516 with = item.with;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
517 when = item.when or dt.parse(item.attr.stamp);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
518 };
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
519 local current_date = dt.date(meta.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
520 if current_date ~= date then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
521 if xmlfile then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
522 assert(xmlfile:close());
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
523 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
524 if dayitems then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
525 assert(dm.list_store(user .. "@" .. date, host, store, dayitems));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
526 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
527 print(current_date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
528 dayitems = {};
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
529 date = current_date;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
530 table.insert(dates, date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
531 xmlfile = assert(io.open(dm.getpath(user .. "@" .. date, host, store, "xml"), "w"));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
532 end
4671
98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion
Kim Alvefur <zash@zash.se>
parents: 4649
diff changeset
533 -- The property 'stamp_legacy' originally came from mod_offline and
98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion
Kim Alvefur <zash@zash.se>
parents: 4649
diff changeset
534 -- was inherited into the mod_storage_internal archive format for
98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion
Kim Alvefur <zash@zash.se>
parents: 4649
diff changeset
535 -- compat. It should not be needed since Prosody 0.10.0 but could
98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion
Kim Alvefur <zash@zash.se>
parents: 4649
diff changeset
536 -- exist in older data.
2815
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
537 item.attr.stamp, item.attr.stamp_legacy = nil, nil;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
538 local stanza = tostring(st.deserialize(item)) .. "\n";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
539 meta.offset, meta.length = xmlfile:seek(), #stanza;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
540 assert(xmlfile:write(stanza));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
541 table.insert(dayitems, meta);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
542 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
543 assert(xmlfile:close());
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
544 assert(dm.list_store(user .. "@" .. date, host, store, dayitems));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
545 assert(dm.list_store(user, host, store, dates));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
546 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
547 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
548
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
549 local store = arg[4];
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
550 if arg[3] == "internal" then
5741
5232d12eb74d mod_storage_xmlarchive: Pass hostname to converter for converting all users
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
551 for i = 5, #arg do
5232d12eb74d mod_storage_xmlarchive: Pass hostname to converter for converting all users
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
552 local user, host = jid.prepped_split(arg[i]);
5232d12eb74d mod_storage_xmlarchive: Pass hostname to converter for converting all users
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
553 if user then
5232d12eb74d mod_storage_xmlarchive: Pass hostname to converter for converting all users
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
554 print(arg[i]);
5232d12eb74d mod_storage_xmlarchive: Pass hostname to converter for converting all users
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
555 convert(user, host, store);
5232d12eb74d mod_storage_xmlarchive: Pass hostname to converter for converting all users
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
556 else
5232d12eb74d mod_storage_xmlarchive: Pass hostname to converter for converting all users
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
557 -- luacheck: ignore 421/user
5232d12eb74d mod_storage_xmlarchive: Pass hostname to converter for converting all users
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
558 for user in archive.users({ host = host; store = store }) do
5232d12eb74d mod_storage_xmlarchive: Pass hostname to converter for converting all users
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
559 print(user.."@"..host);
5232d12eb74d mod_storage_xmlarchive: Pass hostname to converter for converting all users
Kim Alvefur <zash@zash.se>
parents: 5740
diff changeset
560 convert(user, host, store);
5740
100110d539d3 mod_storage_xmlarchive: Migrate all users/rooms if no JID argument given
Kim Alvefur <zash@zash.se>
parents: 5729
diff changeset
561 end
100110d539d3 mod_storage_xmlarchive: Migrate all users/rooms if no JID argument given
Kim Alvefur <zash@zash.se>
parents: 5729
diff changeset
562 end
2815
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
563 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
564 print("Done");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
565 return 0;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
566 else
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
567 print("Currently only conversion to/from mod_storage_internal is supported");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
568 print("Check out https://modules.prosody.im/mod_migrate");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
569 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
570 end
5740
100110d539d3 mod_storage_xmlarchive: Migrate all users/rooms if no JID argument given
Kim Alvefur <zash@zash.se>
parents: 5729
diff changeset
571 print("prosodyctl mod_storage_xmlarchive convert (from|to) internal (archive|archive2|muc_log) [user@host]");
2815
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
572 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format
Kim Alvefur <zash@zash.se>
parents: 2814
diff changeset
573