prosody-modules: mod_storage_xmlarchive/mod_storage

annotate mod_storage_xmlarchive/mod_storage_xmlarchive.lua @ 3857:8752e5b5dd08

mod_rest/README: Add a section describing the JSON mapping Mostly complete

author	Kim Alvefur <zash@zash.se>
date	Sat, 25 Jan 2020 01:31:49 +0100
parents	bb18a1f5e9d7
children	e9e19b9a6a55

rev	line source
1764 a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	1 -- mod_storage_xmlarchive
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	2 -- Copyright (C) 2015-2017 Kim Alvefur
1764 a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	3 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	4 -- This file is MIT/X11 licensed.
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	5 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	6 -- luacheck: ignore unused self
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	7
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	8 local lfs = require "lfs";
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	9 local dm = require "core.storagemanager".olddm;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	10 local hmac_sha256 = require"util.hashes".hmac_sha256;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	11 local st = require"util.stanza";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	12 local dt = require"util.datetime";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	13 local new_stream = require "util.xmppstream".new;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	14 local empty = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	15
2289 aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	16 if not dm.append_raw then
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	17 module:require"datamanager_append_raw";
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	18 end
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	19
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	20 local archive = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	21 local archive_mt = { __index = archive };
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	22
2420 309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	23 local is_stanza = st.is_stanza or function (s)
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	24 return getmetatable(s) == st.stanza_mt;
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	25 end
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	26
1753 54c8a0cb2996 mod_storage_(archive-capable): Change order of arguments to :append to be the same as return values from :find iterator (see prosody 41725f3df3cc) Kim Alvefur <zash@zash.se> parents: 1752 diff changeset	27 function archive:append(username, _, data, when, with)
2420 309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	28 if not is_stanza(data) then
1739 940a4ab75cec mod_storage_xmlarchive: This module only supports storage of stanzas, log a warning about attempts to do otherwise Kim Alvefur <zash@zash.se> parents: 1738 diff changeset	29 module:log("error", "Attempt to store non-stanza object, traceback: %s", debug.traceback());
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	30 return nil, "unsupported-datatype";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	31 end
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	32
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	33 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	34 data = tostring(data) .. "\n";
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	35
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	36 local day = dt.date(when);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	37 local ok, err = dm.append_raw(username.."@"..day, self.host, self.store, "xml", data);
1969 e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	38 if not ok then
3755 bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't Kim Alvefur <zash@zash.se> parents: 3587 diff changeset	39 -- append_raw, unlike list_append, does not log anything on failure atm, so we do so here
bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't Kim Alvefur <zash@zash.se> parents: 3587 diff changeset	40 module:log("error", "Unable to write to %s storage ('%s') for user: %s@%s",
bb18a1f5e9d7 mod_storage_xmlarchive: Log error writing XML since datamanager doesn't Kim Alvefur <zash@zash.se> parents: 3587 diff changeset	41 self.store, err, username, module.host)
1969 e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	42 return nil, err;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	43 end
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	44
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	45 -- If the day-file is missing then we need to add it to the list of days
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	46 local first_of_day = not lfs.attributes(dm.getpath(username .. "@" .. day, self.host, self.store, "list"));
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	47
2595 307ddebb72e1 mod_storage_xmlarchive: Assume offset to be zero if not included (thanks pep.) Kim Alvefur <zash@zash.se> parents: 2514 diff changeset	48 local offset = ok and err or 0;
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	49
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	50 local id = day .. "-" .. hmac_sha256(username.."@"..day.."+"..offset, data, true):sub(-16);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	51 ok, err = dm.list_append(username.."@"..day, self.host, self.store,
2600 9ec8289f6bb2 mod_storage_xmlarchive: Break long line [luacheck] Kim Alvefur <zash@zash.se> parents: 2595 diff changeset	52 { id = id, when = dt.datetime(when), with = with, offset = offset, length = #data });
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	53 if ok and first_of_day then
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	54 ok, err = dm.list_append(username, self.host, self.store, day);
1969 e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	55 end
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	56 if not ok then
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	57 return nil, err;
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	58 end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	59 return id;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	60 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	61
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	62 function archive:_get_idx(username, id, dates)
2938 f000ba14d531 mod_storage_xmlarchive: Add a debug message to try to track down why all indicies are read in some cases Kim Alvefur <zash@zash.se> parents: 2917 diff changeset	63 module:log("debug", "Looking for item with id %q", id);
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	64 dates = dates or self:dates(username) or empty;
3452 0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format Kim Alvefur <zash@zash.se> parents: 3435 diff changeset	65 local date = id:match("^%d%d%d%d%-%d%d%-%d%d");
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	66 for d = 1, #dates do
3452 0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format Kim Alvefur <zash@zash.se> parents: 3435 diff changeset	67 if not date or date == dates[d] then
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	68 module:log("debug", "Loading index for %s", dates[d]);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	69 local items = dm.list_load(username .. "@" .. dates[d], self.host, self.store) or empty;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	70 for i = 1, #items do
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	71 if items[i].id == id then
3462 d73ed7975d82 mod_storage_xmlarchive: Add some debug logging Kim Alvefur <zash@zash.se> parents: 3453 diff changeset	72 module:log("debug", "Found item!");
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	73 return d, i, items;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	74 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	75 end
3453 41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive Kim Alvefur <zash@zash.se> parents: 3452 diff changeset	76 if date then
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive Kim Alvefur <zash@zash.se> parents: 3452 diff changeset	77 return; -- Assuming no duplicates
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive Kim Alvefur <zash@zash.se> parents: 3452 diff changeset	78 end
3452 0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format Kim Alvefur <zash@zash.se> parents: 3435 diff changeset	79 elseif date and date < dates[d] then
3462 d73ed7975d82 mod_storage_xmlarchive: Add some debug logging Kim Alvefur <zash@zash.se> parents: 3453 diff changeset	80 module:log("debug", "Skipping remaining dates after %s", date);
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	81 return; -- List is assumed to be sorted
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	82 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	83 end
3462 d73ed7975d82 mod_storage_xmlarchive: Add some debug logging Kim Alvefur <zash@zash.se> parents: 3453 diff changeset	84 module:log("debug", "Item not found");
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	85 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	86
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	87 function archive:find(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	88 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	89 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	90
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	91 local result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	92 local function cb(_, stanza)
2514 d47a7e6e9adc mod_storage_xmlarchive: Raise error instead of warning if there's ever more than one stanza in a chunk (this indicates some kind of corruption) Kim Alvefur <zash@zash.se> parents: 2420 diff changeset	93 assert(not result, "Multiple items in chunk");
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	94 result = stanza;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	95 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	96
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	97 local stream_session = { notopen = true };
2601 ba5757dc883d mod_storage_xmlarchive: Move XML callback table into a variable to break long line [luacheck] Kim Alvefur <zash@zash.se> parents: 2600 diff changeset	98 local stream_callbacks = { handlestanza = cb, stream_ns = "jabber:client", default_ns = "jabber:client" };
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	99 local stream = new_stream(stream_session, stream_callbacks);
2675 101a2a0b8b33 mod_storage_xmlarchive: Pass username to dates Kim Alvefur <zash@zash.se> parents: 2663 diff changeset	100 local dates = self:dates(username) or empty;
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	101 local function reset_stream()
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	102 stream:reset();
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	103 stream_session.notopen = true;
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	104 stream:feed(st.stanza("stream", { xmlns = "jabber:client" }):top_tag());
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	105 stream_session.notopen = nil;
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	106 end
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	107 reset_stream();
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	108
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	109 local limit = query.limit;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	110 local start_day, step, last_day = 1, 1, #dates;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	111 local count = 0;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	112 local rev = query.reverse;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	113 if query.start then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	114 local d = dt.date(query.start);
3587 c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	115 for i = start_day, last_day, step do
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	116 if dates[i] < d then
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	117 start_day = i + 1;
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	118 elseif dates[i] >= d then
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	119 start_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	120 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	121 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	122 end
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	123 if query["end"] then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	124 local d = dt.date(query["end"]);
3587 c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	125 for i = last_day, start_day, -step do
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	126 if dates[i] > d then
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	127 last_day = i - 1;
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	128 elseif dates[i] <= d then
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	129 last_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	130 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	131 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	132 end
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	133 local items;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	134 local first_item, last_item;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	135 if rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	136 start_day, step, last_day = last_day, -step, start_day;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	137 if query.before then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	138 local before_day, before_item, items_ = self:_get_idx(username, query.before, dates);
3572 7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	139 if not before_day then
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	140 return nil, "item-not-found";
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	141 elseif before_day <= start_day then
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	142 if before_item then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	143 first_item = before_item - 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	144 else
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	145 first_item = #items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	146 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	147 last_item = 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	148 start_day = before_day;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	149 items = items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	150 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	151 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	152 elseif query.after then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	153 local after_day, after_item, items_ = self:_get_idx(username, query.after, dates);
3572 7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	154 if not after_day then
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	155 return nil, "item-not-found";
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	156 elseif after_day >= start_day then
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	157 if after_item then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	158 first_item = after_item + 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	159 else
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	160 first_item = 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	161 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	162 last_item = #items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	163 start_day = after_day;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	164 items = items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	165 end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	166 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	167
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	168 local date_open, xmlfile;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	169 local function read_xml(date, offset, length)
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	170 if xmlfile and date ~= date_open then
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	171 module:log("debug", "Closing XML file for %s", date_open);
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	172 xmlfile:close();
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	173 xmlfile = nil;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	174 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	175 if not xmlfile then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	176 date_open = date;
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	177 local filename = dm.getpath(username .. "@" .. date, self.host, self.store, "xml");
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	178 local ferr;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	179 xmlfile, ferr = io.open(filename);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	180 if not xmlfile then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	181 module:log("error", "Error: %s", ferr);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	182 return nil, ferr;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	183 end
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	184 module:log("debug", "Opened XML file %s", filename);
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	185 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	186 local pos, err = xmlfile:seek("set", offset);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	187 if pos ~= offset then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	188 return nil, err or "seek-failed";
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	189 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	190 return xmlfile:read(length);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	191 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	192
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	193 return function ()
2054 b7c528027762 mod_storage_xmlarchive: Only try to close xmlfile if it has been opened (fixes traceback with limit=0) Kim Alvefur <zash@zash.se> parents: 2041 diff changeset	194 if limit and count >= limit then if xmlfile then xmlfile:close() end return; end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	195 for d = start_day, last_day, step do
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	196 local date = dates[d];
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	197 if not items then
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	198 module:log("debug", "Loading index for %s", date);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	199 start_day = d;
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	200 items = dm.list_load(username .. "@" .. date, self.host, self.store) or empty;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	201 if not rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	202 first_item, last_item = 1, #items;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	203 else
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	204 first_item, last_item = #items, 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	205 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	206 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	207
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	208 local q_with, q_start, q_end = query.with, query.start, query["end"];
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	209 for i = first_item, last_item, step do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	210 local item = items[i];
2396 544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields Kim Alvefur <zash@zash.se> parents: 2290 diff changeset	211 if not item then
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	212 module:log("warn", "data[%q][%d] is nil", date, i);
2396 544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields Kim Alvefur <zash@zash.se> parents: 2290 diff changeset	213 break;
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields Kim Alvefur <zash@zash.se> parents: 2290 diff changeset	214 end
2634 71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability Kim Alvefur <zash@zash.se> parents: 2633 diff changeset	215
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	216 local i_when, i_with = item.when, item.with;
2634 71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability Kim Alvefur <zash@zash.se> parents: 2633 diff changeset	217
1742 ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	218 if type(i_when) == "string" then
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	219 i_when = dt.parse(i_when);
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	220 end
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	221 if type(i_when) ~= "number" then
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	222 module:log("warn", "data[%q][%d].when is invalid", date, i);
1742 ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	223 break;
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	224 end
2634 71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability Kim Alvefur <zash@zash.se> parents: 2633 diff changeset	225
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	226 if (not q_with or i_with == q_with)
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	227 and (not q_start or i_when >= q_start)
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	228 and (not q_end or i_when <= q_end) then
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	229 count = count + 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	230 first_item = i + step;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	231
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	232 local data = read_xml(date, item.offset, item.length);
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	233 if not data then return end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	234 local ok, err = stream:feed(data);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	235 if not ok then
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	236 module:log("warn", "Parse error in %s@%s/%s/%q[%d]: %s", username, self.host, self.store, i, err);
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	237 reset_stream();
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	238 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	239 if result then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	240 local stanza = result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	241 result = nil;
1752 3f3689a16133 mod_storage_xmlarchive: Return 'when' as number Kim Alvefur <zash@zash.se> parents: 1742 diff changeset	242 return item.id, stanza, i_when, i_with;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	243 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	244 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	245 end
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	246 items = nil;
2397 7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	247 if xmlfile then
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	248 xmlfile:close();
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	249 xmlfile = nil;
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	250 end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	251 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	252 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	253 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	254
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	255 function archive:delete(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	256 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	257 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	258 if query.with or query.start or query.after then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	259 return nil, "not-implemented"; -- Only trimming the oldest messages
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	260 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	261 local before = query.before or query["end"] or "9999-12-31";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	262 if type(before) == "number" then before = dt.date(before); else before = before:sub(1, 10); end
2663 6e8c3fc48237 mod_storage_xmlarchive: Use the dates method in more places Kim Alvefur <zash@zash.se> parents: 2634 diff changeset	263 local dates, err = self:dates(username);
2040 459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	264 if not dates or next(dates) == nil then
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	265 if not err then return true end -- already empty
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	266 return dates, err;
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	267 end
2041 7c61ab512d0b mod_storage_xmlarchive: Return earlier if attempting to delete messages older than the oldest existing Kim Alvefur <zash@zash.se> parents: 2040 diff changeset	268 if dates[1] > before then return true; end -- Nothing to delete
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	269 local remaining_dates = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	270 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	271 if dates[d] >= before then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	272 table.insert(remaining_dates, dates[d]);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	273 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	274 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	275 table.sort(remaining_dates);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	276 local ok, err = dm.list_store(username, self.host, self.store, remaining_dates);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	277 if not ok then return ok, err; end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	278 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	279 if dates[d] < before then
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	280 os.remove(dm.getpath(username .. "@" .. dates[d], self.host, self.store, "list"));
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	281 os.remove(dm.getpath(username .. "@" .. dates[d], self.host, self.store, "xml"));
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	282 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	283 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	284 return true;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	285 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	286
1831 004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	287 function archive:dates(username)
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	288 module:log("debug", "Loading root index for %s", username);
2814 1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	289 local dates, err = dm.list_load(username, self.host, self.store);
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	290 if not dates then return dates, err; end
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	291 assert(type(dates[1]) == "string" and type(dates[#dates]) == "string",
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	292 "Archive does not appear to be in xmlarchive format");
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	293 return dates;
1831 004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	294 end
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	295
3585 ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	296 function archive:users()
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	297 return dm.users(module.host, self.store, "list");
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	298 end
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	299
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	300 local provider = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	301 function provider:open(store, typ)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	302 if typ ~= "archive" then return nil, "unsupported-store"; end
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	303 return setmetatable({ host = module.host, store = store }, archive_mt);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	304 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	305
1819 1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	306 function provider:purge(username)
2676 1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	307 local encoded_username = dm.path_encode((username or "@") .. "@");
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	308 local basepath = prosody.paths.data .. "/" .. dm.path_encode(module.host);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	309 for store in lfs.dir(basepath) do
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	310 store = basepath .. "/" .. dm.path_encode(store);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	311 if lfs.attributes(store, "mode") == "directory" then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	312 for file in lfs.dir(store) do
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	313 if file:sub(1, #encoded_username) == encoded_username then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	314 if file:sub(-4) == ".xml" or file:sub(-5) == ".list" then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	315 os.remove(store .. "/" .. file);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	316 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	317 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	318 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	319 return true;
1819 1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	320 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	321 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	322 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	323
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	324 module:provides("storage", provider);
2815 d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	325
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	326
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	327 function module.command(arg)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	328 local jid = require "util.jid";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	329 if arg[1] == "convert" and (arg[2] == "to" or arg[2] == "from") and arg[4] then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	330 local convert;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	331 if arg[2] == "to" then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	332 local xml = require "util.xml";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	333 function convert(user, host, store)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	334 local dates, err = archive.dates({ host = host, store = store }, user);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	335 if not dates then assert(not err, err); return end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	336 assert(dm.list_store(user, host, store, nil));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	337 for _, date in ipairs(dates) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	338 print(date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	339 local items = assert(dm.list_load(user .. "@" .. date, host, store));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	340 local xmlfile = assert(io.open(dm.getpath(user .. "@" .. date, host, store, "xml")));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	341 for _, item in ipairs(items) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	342 assert(xmlfile:seek("set", item.offset));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	343 local data = assert(xmlfile:read(item.length));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	344 assert(#data == item.length, "short read");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	345 data = assert(xml.parse(data));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	346 data = st.preserialize(data);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	347 data.key = item.id;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	348 data.with = item.with;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	349 data.when = tonumber(item.when) or dt.parse(item.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	350 data.attr.stamp = item.when;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	351 data.attr.stamp_legacy = dt.legacy(data.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	352 assert(dm.list_append(user, host, store, data));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	353 end
3491 4e9d4b07e3e9 mod_storage_xmlarchive: Remove per day files during migration Kim Alvefur <zash@zash.se> parents: 3462 diff changeset	354 assert(os.remove(dm.getpath(user .. "@" .. date, host, store, "list")));
4e9d4b07e3e9 mod_storage_xmlarchive: Remove per day files during migration Kim Alvefur <zash@zash.se> parents: 3462 diff changeset	355 assert(os.remove(dm.getpath(user .. "@" .. date, host, store, "xml")));
2815 d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	356 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	357 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	358 else -- convert from internal
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	359 function convert(user, host, store)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	360 local items, err = dm.list_load(user, host, store);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	361 if not items then assert(not err, err); return end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	362 local dates = {};
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	363 local dayitems, date, xmlfile;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	364 for _, item in ipairs(items) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	365 local meta = {
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	366 id = item.key;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	367 with = item.with;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	368 when = item.when or dt.parse(item.attr.stamp);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	369 };
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	370 local current_date = dt.date(meta.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	371 if current_date ~= date then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	372 if xmlfile then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	373 assert(xmlfile:close());
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	374 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	375 if dayitems then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	376 assert(dm.list_store(user .. "@" .. date, host, store, dayitems));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	377 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	378 print(current_date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	379 dayitems = {};
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	380 date = current_date;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	381 table.insert(dates, date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	382 xmlfile = assert(io.open(dm.getpath(user .. "@" .. date, host, store, "xml"), "w"));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	383 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	384 item.attr.stamp, item.attr.stamp_legacy = nil, nil;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	385 local stanza = tostring(st.deserialize(item)) .. "\n";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	386 meta.offset, meta.length = xmlfile:seek(), #stanza;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	387 assert(xmlfile:write(stanza));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	388 table.insert(dayitems, meta);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	389 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	390 assert(xmlfile:close());
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	391 assert(dm.list_store(user .. "@" .. date, host, store, dayitems));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	392 assert(dm.list_store(user, host, store, dates));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	393 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	394 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	395
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	396 local store = arg[4];
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	397 if arg[3] == "internal" then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	398 for i = 5, #arg do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	399 local user, host = jid.prepped_split(arg[i]);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	400 if not user then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	401 print(string.format("Argument #%d (%q) is an invalid JID, aborting", i, arg[i]));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	402 os.exit(1);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	403 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	404 convert(user, host, store);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	405 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	406 print("Done");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	407 return 0;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	408 else
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	409 print("Currently only conversion to/from mod_storage_internal is supported");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	410 print("Check out https://modules.prosody.im/mod_migrate");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	411 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	412 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	413 print("prosodyctl mod_storage_xmlarchive convert (from\|to) internal (archive\|archive2\|muc_log) user@host");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	414 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	415

Mercurial > prosody-modules

annotate mod_storage_xmlarchive/mod_storage_xmlarchive.lua @ 3857:8752e5b5dd08