prosody-modules: mod_storage_xmlarchive/mod_storage

annotate mod_storage_xmlarchive/mod_storage_xmlarchive.lua @ 4688:05725276fac0

mod_bookmarks2: Use same default as mod_pep for max_items Should fix the issue with max items until the proper "max" can be used, by following the configured max. While "max" is already in trunk, it's not easily usable in 0.11.x This limit and option was added to mod_pep in Prosody rev aefb96a52f5f

author	Kim Alvefur <zash@zash.se>
date	Wed, 15 Sep 2021 17:39:37 +0200
parents	98bf0f597df4
children	79ba1a1a75cc

rev	line source
1764 a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	1 -- mod_storage_xmlarchive
4647 b91e40472d68 mod_storage_xmlarchive: Fix return of numeric 'when' from key-value API Kim Alvefur <zash@zash.se> parents: 4646 diff changeset	2 -- Copyright (C) 2015-2021 Kim Alvefur
1764 a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	3 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	4 -- This file is MIT/X11 licensed.
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	5 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	6 -- luacheck: ignore unused self
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	7
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	8 local lfs = require "lfs";
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	9 local dm = require "core.storagemanager".olddm;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	10 local hmac_sha256 = require"util.hashes".hmac_sha256;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	11 local st = require"util.stanza";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	12 local dt = require"util.datetime";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	13 local new_stream = require "util.xmppstream".new;
3898 e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	14 local xml = require "util.xml";
4538 591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	15 local async = require "util.async";
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	16 local empty = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	17
2289 aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	18 if not dm.append_raw then
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	19 module:require"datamanager_append_raw";
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	20 end
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	21
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	22 local archive = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	23 local archive_mt = { __index = archive };
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	24
4646 072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm) Kim Alvefur <zash@zash.se> parents: 4551 diff changeset	25 archive.caps = {
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm) Kim Alvefur <zash@zash.se> parents: 4551 diff changeset	26 total = false,
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm) Kim Alvefur <zash@zash.se> parents: 4551 diff changeset	27 quota = nil,
4648 37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	28 full_id_range = true;
4649 62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	29 ids = true;
4646 072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm) Kim Alvefur <zash@zash.se> parents: 4551 diff changeset	30 };
072d078be095 mod_storage_xmlarchive: Advertise capabilities (none atm) Kim Alvefur <zash@zash.se> parents: 4551 diff changeset	31
2420 309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	32 local is_stanza = st.is_stanza or function (s)
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	33 return getmetatable(s) == st.stanza_mt;
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	34 end
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	35
1753 54c8a0cb2996 mod_storage_(archive-capable): Change order of arguments to :append to be the same as return values from :find iterator (see prosody 41725f3df3cc) Kim Alvefur <zash@zash.se> parents: 1752 diff changeset	36 function archive:append(username, _, data, when, with)
2420 309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	37 if not is_stanza(data) then
1739 940a4ab75cec mod_storage_xmlarchive: This module only supports storage of stanzas, log a warning about attempts to do otherwise Kim Alvefur <zash@zash.se> parents: 1738 diff changeset	38 module:log("error", "Attempt to store non-stanza object, traceback: %s", debug.traceback());
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	39 return nil, "unsupported-datatype";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	40 end
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	41
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	42 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	43 data = tostring(data) .. "\n";
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	44
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	45 local day = dt.date(when);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	46 local ok, err = dm.append_raw(username.."@"..day, self.host, self.store, "xml", data);
1969 e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	47 if not ok then
3755 bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't Kim Alvefur <zash@zash.se> parents: 3587 diff changeset	48 -- append_raw, unlike list_append, does not log anything on failure atm, so we do so here
bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't Kim Alvefur <zash@zash.se> parents: 3587 diff changeset	49 module:log("error", "Unable to write to %s storage ('%s') for user: %s@%s",
bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't Kim Alvefur <zash@zash.se> parents: 3587 diff changeset	50 self.store, err, username, module.host)
1969 e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	51 return nil, err;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	52 end
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	53
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	54 -- If the day-file is missing then we need to add it to the list of days
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	55 local first_of_day = not lfs.attributes(dm.getpath(username .. "@" .. day, self.host, self.store, "list"));
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	56
2595 307ddebb72e1 mod_storage_xmlarchive: Assume offset to be zero if not included (thanks pep.) Kim Alvefur <zash@zash.se> parents: 2514 diff changeset	57 local offset = ok and err or 0;
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	58
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	59 local id = day .. "-" .. hmac_sha256(username.."@"..day.."+"..offset, data, true):sub(-16);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	60 ok, err = dm.list_append(username.."@"..day, self.host, self.store,
2600 9ec8289f6bb2 mod_storage_xmlarchive: Break long line [luacheck] Kim Alvefur <zash@zash.se> parents: 2595 diff changeset	61 { id = id, when = dt.datetime(when), with = with, offset = offset, length = #data });
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	62 if ok and first_of_day then
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	63 ok, err = dm.list_append(username, self.host, self.store, day);
1969 e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	64 end
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	65 if not ok then
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	66 return nil, err;
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	67 end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	68 return id;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	69 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	70
3898 e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	71 function archive:get(username, id)
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	72 local dates = self:dates(username) or empty;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	73 local day_idx, item_idx, items = self:_get_idx(username, id, dates);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	74 if not day_idx then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	75 return nil, "item-not-found";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	76 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	77 module:log("debug", ":_get_idx(%q, %q) --> %q, %q", username, id, dates[day_idx], items[item_idx]);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	78 local day = dates[day_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	79 local item = items[item_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	80 module:log("debug", "item = %q", item);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	81 local filename = dm.getpath(username.."@"..day, self.host, self.store, "xml");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	82 local xmlfile, ferr = io.open(filename, "r");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	83 if not xmlfile then return nil, ferr; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	84 local p,err = xmlfile:seek("set", item.offset);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	85 if p ~= item.offset or err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	86 local data = xmlfile:read(item.length);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	87 local parsed, perr = xml.parse(data);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	88 if not parsed then return nil, perr; end
4647 b91e40472d68 mod_storage_xmlarchive: Fix return of numeric 'when' from key-value API Kim Alvefur <zash@zash.se> parents: 4646 diff changeset	89 return parsed, tonumber(item.when) or dt.parse(item.when), item.with;
3898 e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	90 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	91
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	92 local overwrite = module:get_option("xmlarchive_overwrite", false);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	93
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	94 function archive:set(username, id, data, new_when, new_with)
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	95 if not is_stanza(data) then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	96 module:log("error", "Attempt to store non-stanza object, traceback: %s", debug.traceback());
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	97 return nil, "unsupported-datatype";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	98 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	99
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	100 username = username or "@";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	101 data = tostring(data) .. "\n";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	102
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	103 local dates = self:dates(username) or empty;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	104 local day_idx, item_idx, items = self:_get_idx(username, id, dates);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	105 if not day_idx then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	106 return nil, "item-not-found";
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	107 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	108 local day = dates[day_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	109 local item = items[item_idx];
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	110
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	111 local filename = dm.getpath(username.."@"..day, self.host, self.store, "xml");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	112
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	113 local replaced, err = dm.append_raw(username.."@"..day, self.host, self.store, "xml", data);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	114 if not replaced then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	115 local new_offset = err;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	116
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	117 -- default yes or no?
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	118 if overwrite then
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	119 local xmlfile, ferr = io.open(filename, "r+");
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	120 if not xmlfile then return nil, ferr; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	121 local p,err = xmlfile:seek("set", item.offset);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	122 if p ~= item.offset or err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	123 local _,err = xmlfile:write((" "):rep(item.length));
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	124 if err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	125 local _,err = xmlfile:close();
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	126 if err ~= nil then return nil, err; end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	127 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	128
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	129 items[item_idx] = {
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	130 id = id,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	131 when = new_when or item.when,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	132 with = new_with or item.with,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	133 offset = new_offset,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	134 length = #data,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	135 replaces = item,
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	136 };
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	137 local ok, err = dm.list_store(username.."@"..day, self.host, self.store, items);
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	138 return ok, err;
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	139 end
e9e19b9a6a55 mod_storage_xmlarchive: Add a new API similar to map-stores Kim Alvefur <zash@zash.se> parents: 3755 diff changeset	140
4538 591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	141 local function get_nexttick()
4551 b92147edd172 mod_storage_xmlarchive: Workaround for #1646 (util.async bug with Lua 5.1) Kim Alvefur <zash@zash.se> parents: 4538 diff changeset	142 -- COMPAT util.async under Lua 5.1 runs into problems trying to yield across
b92147edd172 mod_storage_xmlarchive: Workaround for #1646 (util.async bug with Lua 5.1) Kim Alvefur <zash@zash.se> parents: 4538 diff changeset	143 -- pcall barriers. Workaround for #1646
b92147edd172 mod_storage_xmlarchive: Workaround for #1646 (util.async bug with Lua 5.1) Kim Alvefur <zash@zash.se> parents: 4538 diff changeset	144 if _VERSION ~= "Lua 5.1" and async.ready() then
4538 591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	145 return function ()
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	146 -- slow down
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	147 local wait, done = async.waiter();
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	148 module:add_timer(0, done);
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	149 wait();
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	150 end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	151 else
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	152 -- no async, no-op
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	153 return function () end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	154 end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	155 end
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	156
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	157 function archive:_get_idx(username, id, dates)
2938 f000ba14d531 mod_storage_xmlarchive: Add a debug message to try to track down why all indicies are read in some cases Kim Alvefur <zash@zash.se> parents: 2917 diff changeset	158 module:log("debug", "Looking for item with id %q", id);
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	159 dates = dates or self:dates(username) or empty;
3452 0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format Kim Alvefur <zash@zash.se> parents: 3435 diff changeset	160 local date = id:match("^%d%d%d%d%-%d%d%-%d%d");
4538 591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	161 local tick = get_nexttick();
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	162 for d = 1, #dates do
3452 0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format Kim Alvefur <zash@zash.se> parents: 3435 diff changeset	163 if not date or date == dates[d] then
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	164 module:log("debug", "Loading index for %s", dates[d]);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	165 local items = dm.list_load(username .. "@" .. dates[d], self.host, self.store) or empty;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	166 for i = 1, #items do
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	167 if items[i].id == id then
3462 d73ed7975d82 mod_storage_xmlarchive: Add some debug logging Kim Alvefur <zash@zash.se> parents: 3453 diff changeset	168 module:log("debug", "Found item!");
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	169 return d, i, items;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	170 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	171 end
3453 41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive Kim Alvefur <zash@zash.se> parents: 3452 diff changeset	172 if date then
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive Kim Alvefur <zash@zash.se> parents: 3452 diff changeset	173 return; -- Assuming no duplicates
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive Kim Alvefur <zash@zash.se> parents: 3452 diff changeset	174 end
3452 0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format Kim Alvefur <zash@zash.se> parents: 3435 diff changeset	175 elseif date and date < dates[d] then
3462 d73ed7975d82 mod_storage_xmlarchive: Add some debug logging Kim Alvefur <zash@zash.se> parents: 3453 diff changeset	176 module:log("debug", "Skipping remaining dates after %s", date);
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	177 return; -- List is assumed to be sorted
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	178 end
4538 591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	179
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	180 -- insert pauses to allow other processing
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	181 if d % 14 == 0 then tick(); end
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	182 end
3462 d73ed7975d82 mod_storage_xmlarchive: Add some debug logging Kim Alvefur <zash@zash.se> parents: 3453 diff changeset	183 module:log("debug", "Item not found");
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	184 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	185
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	186 function archive:find(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	187 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	188 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	189
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	190 local result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	191 local function cb(_, stanza)
2514 d47a7e6e9adc mod_storage_xmlarchive: Raise error instead of warning if there's ever more than one stanza in a chunk (this indicates some kind of corruption) Kim Alvefur <zash@zash.se> parents: 2420 diff changeset	192 assert(not result, "Multiple items in chunk");
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	193 result = stanza;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	194 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	195
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	196 local stream_session = { notopen = true };
2601 ba5757dc883d mod_storage_xmlarchive: Move XML callback table into a variable to break long line [luacheck] Kim Alvefur <zash@zash.se> parents: 2600 diff changeset	197 local stream_callbacks = { handlestanza = cb, stream_ns = "jabber:client", default_ns = "jabber:client" };
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	198 local stream = new_stream(stream_session, stream_callbacks);
2675 101a2a0b8b33 mod_storage_xmlarchive: Pass username to dates Kim Alvefur <zash@zash.se> parents: 2663 diff changeset	199 local dates = self:dates(username) or empty;
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	200 local function reset_stream()
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	201 stream:reset();
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	202 stream_session.notopen = true;
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	203 stream:feed(st.stanza("stream", { xmlns = "jabber:client" }):top_tag());
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	204 stream_session.notopen = nil;
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	205 end
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	206 reset_stream();
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	207
4649 62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	208 local filtered_ids = false;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	209 local filtered_dates = false;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	210 if query.ids then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	211 filtered_ids = {};
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	212 filtered_dates = {};
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	213 for _, id in ipairs(query.ids) do
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	214 filtered_ids[id] = true;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	215 if filtered_dates then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	216 local date = id:match("^%d%d%d%d%-%d%d%-%d%d");
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	217 if date then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	218 filtered_dates[date] = true;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	219 else
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	220 -- if any id diverges from the standard then the item could be from any date
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	221 filtered_dates = nil;
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	222 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	223 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	224 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	225 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	226
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	227 if filtered_dates then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	228 for i = #dates, 1, -1 do
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	229 if not filtered_dates[ dates[i] ] then
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	230 table.remove(dates, i);
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	231 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	232 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	233 end
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	234
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	235 local limit = query.limit;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	236 local start_day, step, last_day = 1, 1, #dates;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	237 local count = 0;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	238 local rev = query.reverse;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	239 if query.start then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	240 local d = dt.date(query.start);
3587 c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	241 for i = start_day, last_day, step do
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	242 if dates[i] < d then
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	243 start_day = i + 1;
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	244 elseif dates[i] >= d then
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	245 start_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	246 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	247 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	248 end
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	249 if query["end"] then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	250 local d = dt.date(query["end"]);
3587 c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	251 for i = last_day, start_day, -step do
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	252 if dates[i] > d then
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	253 last_day = i - 1;
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	254 elseif dates[i] <= d then
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	255 last_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	256 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	257 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	258 end
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	259 local items;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	260 local first_item, last_item;
4648 37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	261 local stop_at_id;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	262 if rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	263 start_day, step, last_day = last_day, -step, start_day;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	264 if query.before then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	265 local before_day, before_item, items_ = self:_get_idx(username, query.before, dates);
3572 7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	266 if not before_day then
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	267 return nil, "item-not-found";
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	268 elseif before_day <= start_day then
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	269 if before_item then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	270 first_item = before_item - 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	271 else
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	272 first_item = #items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	273 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	274 last_item = 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	275 start_day = before_day;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	276 items = items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	277 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	278 end
4648 37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	279 if query.after then
37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	280 stop_at_id = query.after;
37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	281 end
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	282 elseif query.after then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	283 local after_day, after_item, items_ = self:_get_idx(username, query.after, dates);
3572 7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	284 if not after_day then
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	285 return nil, "item-not-found";
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	286 elseif after_day >= start_day then
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	287 if after_item then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	288 first_item = after_item + 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	289 else
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	290 first_item = 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	291 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	292 last_item = #items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	293 start_day = after_day;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	294 items = items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	295 end
4648 37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	296 if query.before then
37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	297 stop_at_id = query.before;
37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	298 end
37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	299 elseif query.before then
37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	300 stop_at_id = query.before;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	301 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	302
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	303 local date_open, xmlfile;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	304 local function read_xml(date, offset, length)
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	305 if xmlfile and date ~= date_open then
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	306 module:log("debug", "Closing XML file for %s", date_open);
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	307 xmlfile:close();
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	308 xmlfile = nil;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	309 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	310 if not xmlfile then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	311 date_open = date;
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	312 local filename = dm.getpath(username .. "@" .. date, self.host, self.store, "xml");
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	313 local ferr;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	314 xmlfile, ferr = io.open(filename);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	315 if not xmlfile then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	316 module:log("error", "Error: %s", ferr);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	317 return nil, ferr;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	318 end
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	319 module:log("debug", "Opened XML file %s", filename);
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	320 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	321 local pos, err = xmlfile:seek("set", offset);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	322 if pos ~= offset then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	323 return nil, err or "seek-failed";
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	324 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	325 return xmlfile:read(length);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	326 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	327
4538 591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	328 local tick = get_nexttick();
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	329
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	330 return function ()
2054 b7c528027762 mod_storage_xmlarchive: Only try to close xmlfile if it has been opened (fixes traceback with limit=0) Kim Alvefur <zash@zash.se> parents: 2041 diff changeset	331 if limit and count >= limit then if xmlfile then xmlfile:close() end return; end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	332 for d = start_day, last_day, step do
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	333 local date = dates[d];
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	334 if not items then
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	335 module:log("debug", "Loading index for %s", date);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	336 start_day = d;
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	337 items = dm.list_load(username .. "@" .. date, self.host, self.store) or empty;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	338 if not rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	339 first_item, last_item = 1, #items;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	340 else
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	341 first_item, last_item = #items, 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	342 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	343 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	344
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	345 local q_with, q_start, q_end = query.with, query.start, query["end"];
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	346 for i = first_item, last_item, step do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	347 local item = items[i];
2396 544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields Kim Alvefur <zash@zash.se> parents: 2290 diff changeset	348 if not item then
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	349 module:log("warn", "data[%q][%d] is nil", date, i);
2396 544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields Kim Alvefur <zash@zash.se> parents: 2290 diff changeset	350 break;
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields Kim Alvefur <zash@zash.se> parents: 2290 diff changeset	351 end
2634 71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability Kim Alvefur <zash@zash.se> parents: 2633 diff changeset	352
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	353 local i_when, i_with = item.when, item.with;
2634 71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability Kim Alvefur <zash@zash.se> parents: 2633 diff changeset	354
1742 ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	355 if type(i_when) == "string" then
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	356 i_when = dt.parse(i_when);
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	357 end
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	358 if type(i_when) ~= "number" then
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	359 module:log("warn", "data[%q][%d].when is invalid", date, i);
1742 ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	360 break;
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	361 end
2634 71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability Kim Alvefur <zash@zash.se> parents: 2633 diff changeset	362
4648 37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	363 if stop_at_id and stop_at_id == item.id then
37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	364 if xmlfile then xmlfile:close(); end
37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	365 return;
37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	366 end
37ca6109077f mod_storage_xmlarchive: Support full id range queries Kim Alvefur <zash@zash.se> parents: 4647 diff changeset	367
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	368 if (not q_with or i_with == q_with)
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	369 and (not q_start or i_when >= q_start)
4649 62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	370 and (not q_end or i_when <= q_end)
62d41447615d mod_storage_xmlarchive: Support query by list of IDs Kim Alvefur <zash@zash.se> parents: 4648 diff changeset	371 and (not filtered_ids or filtered_ids[item.id]) then
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	372 count = count + 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	373 first_item = i + step;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	374
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	375 local data = read_xml(date, item.offset, item.length);
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	376 if not data then return end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	377 local ok, err = stream:feed(data);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	378 if not ok then
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	379 module:log("warn", "Parse error in %s@%s/%s/%q[%d]: %s", username, self.host, self.store, i, err);
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	380 reset_stream();
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	381 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	382 if result then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	383 local stanza = result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	384 result = nil;
1752 3f3689a16133 mod_storage_xmlarchive: Return 'when' as number Kim Alvefur <zash@zash.se> parents: 1742 diff changeset	385 return item.id, stanza, i_when, i_with;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	386 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	387 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	388 end
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	389 items = nil;
2397 7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	390 if xmlfile then
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	391 xmlfile:close();
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	392 xmlfile = nil;
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	393 end
4538 591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	394 -- If we're running through a lot of day-files then lets allow for other processing between each day
591c643d55b2 mod_storage_xmlarchive: Insert micropauses in long-running queries Kim Alvefur <zash@zash.se> parents: 3899 diff changeset	395 tick();
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	396 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	397 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	398 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	399
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	400 function archive:delete(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	401 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	402 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	403 if query.with or query.start or query.after then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	404 return nil, "not-implemented"; -- Only trimming the oldest messages
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	405 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	406 local before = query.before or query["end"] or "9999-12-31";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	407 if type(before) == "number" then before = dt.date(before); else before = before:sub(1, 10); end
2663 6e8c3fc48237 mod_storage_xmlarchive: Use the dates method in more places Kim Alvefur <zash@zash.se> parents: 2634 diff changeset	408 local dates, err = self:dates(username);
2040 459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	409 if not dates or next(dates) == nil then
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	410 if not err then return true end -- already empty
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	411 return dates, err;
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	412 end
2041 7c61ab512d0b mod_storage_xmlarchive: Return earlier if attempting to delete messages older than the oldest existing Kim Alvefur <zash@zash.se> parents: 2040 diff changeset	413 if dates[1] > before then return true; end -- Nothing to delete
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	414 local remaining_dates = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	415 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	416 if dates[d] >= before then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	417 table.insert(remaining_dates, dates[d]);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	418 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	419 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	420 table.sort(remaining_dates);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	421 local ok, err = dm.list_store(username, self.host, self.store, remaining_dates);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	422 if not ok then return ok, err; end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	423 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	424 if dates[d] < before then
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	425 os.remove(dm.getpath(username .. "@" .. dates[d], self.host, self.store, "list"));
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	426 os.remove(dm.getpath(username .. "@" .. dates[d], self.host, self.store, "xml"));
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	427 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	428 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	429 return true;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	430 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	431
1831 004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	432 function archive:dates(username)
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	433 module:log("debug", "Loading root index for %s", username);
2814 1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	434 local dates, err = dm.list_load(username, self.host, self.store);
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	435 if not dates then return dates, err; end
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	436 assert(type(dates[1]) == "string" and type(dates[#dates]) == "string",
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	437 "Archive does not appear to be in xmlarchive format");
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	438 return dates;
1831 004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	439 end
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	440
3585 ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	441 function archive:users()
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	442 return dm.users(module.host, self.store, "list");
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	443 end
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	444
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	445 local provider = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	446 function provider:open(store, typ)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	447 if typ ~= "archive" then return nil, "unsupported-store"; end
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	448 return setmetatable({ host = module.host, store = store }, archive_mt);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	449 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	450
1819 1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	451 function provider:purge(username)
2676 1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	452 local encoded_username = dm.path_encode((username or "@") .. "@");
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	453 local basepath = prosody.paths.data .. "/" .. dm.path_encode(module.host);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	454 for store in lfs.dir(basepath) do
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	455 store = basepath .. "/" .. dm.path_encode(store);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	456 if lfs.attributes(store, "mode") == "directory" then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	457 for file in lfs.dir(store) do
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	458 if file:sub(1, #encoded_username) == encoded_username then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	459 if file:sub(-4) == ".xml" or file:sub(-5) == ".list" then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	460 os.remove(store .. "/" .. file);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	461 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	462 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	463 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	464 return true;
1819 1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	465 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	466 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	467 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	468
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	469 module:provides("storage", provider);
2815 d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	470
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	471
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	472 function module.command(arg)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	473 local jid = require "util.jid";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	474 if arg[1] == "convert" and (arg[2] == "to" or arg[2] == "from") and arg[4] then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	475 local convert;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	476 if arg[2] == "to" then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	477 function convert(user, host, store)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	478 local dates, err = archive.dates({ host = host, store = store }, user);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	479 if not dates then assert(not err, err); return end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	480 assert(dm.list_store(user, host, store, nil));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	481 for _, date in ipairs(dates) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	482 print(date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	483 local items = assert(dm.list_load(user .. "@" .. date, host, store));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	484 local xmlfile = assert(io.open(dm.getpath(user .. "@" .. date, host, store, "xml")));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	485 for _, item in ipairs(items) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	486 assert(xmlfile:seek("set", item.offset));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	487 local data = assert(xmlfile:read(item.length));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	488 assert(#data == item.length, "short read");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	489 data = assert(xml.parse(data));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	490 data = st.preserialize(data);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	491 data.key = item.id;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	492 data.with = item.with;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	493 data.when = tonumber(item.when) or dt.parse(item.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	494 data.attr.stamp = item.when;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	495 assert(dm.list_append(user, host, store, data));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	496 end
3491 4e9d4b07e3e9 mod_storage_xmlarchive: Remove per day files during migration Kim Alvefur <zash@zash.se> parents: 3462 diff changeset	497 assert(os.remove(dm.getpath(user .. "@" .. date, host, store, "list")));
4e9d4b07e3e9 mod_storage_xmlarchive: Remove per day files during migration Kim Alvefur <zash@zash.se> parents: 3462 diff changeset	498 assert(os.remove(dm.getpath(user .. "@" .. date, host, store, "xml")));
2815 d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	499 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	500 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	501 else -- convert from internal
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	502 function convert(user, host, store)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	503 local items, err = dm.list_load(user, host, store);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	504 if not items then assert(not err, err); return end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	505 local dates = {};
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	506 local dayitems, date, xmlfile;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	507 for _, item in ipairs(items) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	508 local meta = {
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	509 id = item.key;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	510 with = item.with;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	511 when = item.when or dt.parse(item.attr.stamp);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	512 };
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	513 local current_date = dt.date(meta.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	514 if current_date ~= date then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	515 if xmlfile then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	516 assert(xmlfile:close());
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	517 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	518 if dayitems then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	519 assert(dm.list_store(user .. "@" .. date, host, store, dayitems));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	520 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	521 print(current_date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	522 dayitems = {};
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	523 date = current_date;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	524 table.insert(dates, date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	525 xmlfile = assert(io.open(dm.getpath(user .. "@" .. date, host, store, "xml"), "w"));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	526 end
4671 98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion Kim Alvefur <zash@zash.se> parents: 4649 diff changeset	527 -- The property 'stamp_legacy' originally came from mod_offline and
98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion Kim Alvefur <zash@zash.se> parents: 4649 diff changeset	528 -- was inherited into the mod_storage_internal archive format for
98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion Kim Alvefur <zash@zash.se> parents: 4649 diff changeset	529 -- compat. It should not be needed since Prosody 0.10.0 but could
98bf0f597df4 mod_storage_xmlarchive: Stop exporting XEP-0091 timestamp in conversion Kim Alvefur <zash@zash.se> parents: 4649 diff changeset	530 -- exist in older data.
2815 d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	531 item.attr.stamp, item.attr.stamp_legacy = nil, nil;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	532 local stanza = tostring(st.deserialize(item)) .. "\n";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	533 meta.offset, meta.length = xmlfile:seek(), #stanza;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	534 assert(xmlfile:write(stanza));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	535 table.insert(dayitems, meta);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	536 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	537 assert(xmlfile:close());
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	538 assert(dm.list_store(user .. "@" .. date, host, store, dayitems));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	539 assert(dm.list_store(user, host, store, dates));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	540 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	541 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	542
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	543 local store = arg[4];
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	544 if arg[3] == "internal" then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	545 for i = 5, #arg do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	546 local user, host = jid.prepped_split(arg[i]);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	547 if not user then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	548 print(string.format("Argument #%d (%q) is an invalid JID, aborting", i, arg[i]));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	549 os.exit(1);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	550 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	551 convert(user, host, store);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	552 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	553 print("Done");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	554 return 0;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	555 else
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	556 print("Currently only conversion to/from mod_storage_internal is supported");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	557 print("Check out https://modules.prosody.im/mod_migrate");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	558 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	559 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	560 print("prosodyctl mod_storage_xmlarchive convert (from\|to) internal (archive\|archive2\|muc_log) user@host");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	561 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	562

Mercurial > prosody-modules

annotate mod_storage_xmlarchive/mod_storage_xmlarchive.lua @ 4688:05725276fac0