prosody-modules: mod_storage_xmlarchive/mod_storage

annotate mod_storage_xmlarchive/mod_storage_xmlarchive.lua @ 3665:b105c4425359

mod_profile: Mark as Unmaintained

author	Kim Alvefur <zash@zash.se>
date	Tue, 03 Sep 2019 00:52:47 +0200
parents	c24d43ababc6
children	bb18a1f5e9d7

rev	line source
1764 a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	1 -- mod_storage_xmlarchive
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	2 -- Copyright (C) 2015-2017 Kim Alvefur
1764 a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	3 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	4 -- This file is MIT/X11 licensed.
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	5 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	6 -- luacheck: ignore unused self
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	7
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	8 local lfs = require "lfs";
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	9 local dm = require "core.storagemanager".olddm;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	10 local hmac_sha256 = require"util.hashes".hmac_sha256;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	11 local st = require"util.stanza";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	12 local dt = require"util.datetime";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	13 local new_stream = require "util.xmppstream".new;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	14 local empty = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	15
2289 aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	16 if not dm.append_raw then
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	17 module:require"datamanager_append_raw";
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	18 end
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	19
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	20 local archive = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	21 local archive_mt = { __index = archive };
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	22
2420 309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	23 local is_stanza = st.is_stanza or function (s)
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	24 return getmetatable(s) == st.stanza_mt;
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	25 end
309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	26
1753 54c8a0cb2996 mod_storage_(archive-capable): Change order of arguments to :append to be the same as return values from :find iterator (see prosody 41725f3df3cc) Kim Alvefur <zash@zash.se> parents: 1752 diff changeset	27 function archive:append(username, _, data, when, with)
2420 309db11494c2 mod_storage_xmlarchive: Use util.stanza.is_stanza if available Kim Alvefur <zash@zash.se> parents: 2405 diff changeset	28 if not is_stanza(data) then
1739 940a4ab75cec mod_storage_xmlarchive: This module only supports storage of stanzas, log a warning about attempts to do otherwise Kim Alvefur <zash@zash.se> parents: 1738 diff changeset	29 module:log("error", "Attempt to store non-stanza object, traceback: %s", debug.traceback());
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	30 return nil, "unsupported-datatype";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	31 end
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	32
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	33 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	34 data = tostring(data) .. "\n";
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	35
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	36 local day = dt.date(when);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	37 local ok, err = dm.append_raw(username.."@"..day, self.host, self.store, "xml", data);
1969 e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	38 if not ok then
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	39 return nil, err;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	40 end
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	41
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	42 -- If the day-file is missing then we need to add it to the list of days
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	43 local first_of_day = not lfs.attributes(dm.getpath(username .. "@" .. day, self.host, self.store, "list"));
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	44
2595 307ddebb72e1 mod_storage_xmlarchive: Assume offset to be zero if not included (thanks pep.) Kim Alvefur <zash@zash.se> parents: 2514 diff changeset	45 local offset = ok and err or 0;
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	46
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	47 local id = day .. "-" .. hmac_sha256(username.."@"..day.."+"..offset, data, true):sub(-16);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	48 ok, err = dm.list_append(username.."@"..day, self.host, self.store,
2600 9ec8289f6bb2 mod_storage_xmlarchive: Break long line [luacheck] Kim Alvefur <zash@zash.se> parents: 2595 diff changeset	49 { id = id, when = dt.datetime(when), with = with, offset = offset, length = #data });
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	50 if ok and first_of_day then
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	51 ok, err = dm.list_append(username, self.host, self.store, day);
1969 e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	52 end
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	53 if not ok then
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	54 return nil, err;
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	55 end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	56 return id;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	57 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	58
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	59 function archive:_get_idx(username, id, dates)
2938 f000ba14d531 mod_storage_xmlarchive: Add a debug message to try to track down why all indicies are read in some cases Kim Alvefur <zash@zash.se> parents: 2917 diff changeset	60 module:log("debug", "Looking for item with id %q", id);
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	61 dates = dates or self:dates(username) or empty;
3452 0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format Kim Alvefur <zash@zash.se> parents: 3435 diff changeset	62 local date = id:match("^%d%d%d%d%-%d%d%-%d%d");
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	63 for d = 1, #dates do
3452 0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format Kim Alvefur <zash@zash.se> parents: 3435 diff changeset	64 if not date or date == dates[d] then
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	65 module:log("debug", "Loading index for %s", dates[d]);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	66 local items = dm.list_load(username .. "@" .. dates[d], self.host, self.store) or empty;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	67 for i = 1, #items do
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	68 if items[i].id == id then
3462 d73ed7975d82 mod_storage_xmlarchive: Add some debug logging Kim Alvefur <zash@zash.se> parents: 3453 diff changeset	69 module:log("debug", "Found item!");
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	70 return d, i, items;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	71 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	72 end
3453 41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive Kim Alvefur <zash@zash.se> parents: 3452 diff changeset	73 if date then
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive Kim Alvefur <zash@zash.se> parents: 3452 diff changeset	74 return; -- Assuming no duplicates
41e1cacf3c4e mod_storage_xmlarchive: Don't return early when looking through entire archive Kim Alvefur <zash@zash.se> parents: 3452 diff changeset	75 end
3452 0c539092aa75 mod_storage_xmlarchive: Allow lookup of IDs that don't follow the YYYY-MM-DD-RANDOM format Kim Alvefur <zash@zash.se> parents: 3435 diff changeset	76 elseif date and date < dates[d] then
3462 d73ed7975d82 mod_storage_xmlarchive: Add some debug logging Kim Alvefur <zash@zash.se> parents: 3453 diff changeset	77 module:log("debug", "Skipping remaining dates after %s", date);
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	78 return; -- List is assumed to be sorted
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	79 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	80 end
3462 d73ed7975d82 mod_storage_xmlarchive: Add some debug logging Kim Alvefur <zash@zash.se> parents: 3453 diff changeset	81 module:log("debug", "Item not found");
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	82 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	83
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	84 function archive:find(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	85 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	86 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	87
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	88 local result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	89 local function cb(_, stanza)
2514 d47a7e6e9adc mod_storage_xmlarchive: Raise error instead of warning if there's ever more than one stanza in a chunk (this indicates some kind of corruption) Kim Alvefur <zash@zash.se> parents: 2420 diff changeset	90 assert(not result, "Multiple items in chunk");
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	91 result = stanza;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	92 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	93
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	94 local stream_session = { notopen = true };
2601 ba5757dc883d mod_storage_xmlarchive: Move XML callback table into a variable to break long line [luacheck] Kim Alvefur <zash@zash.se> parents: 2600 diff changeset	95 local stream_callbacks = { handlestanza = cb, stream_ns = "jabber:client", default_ns = "jabber:client" };
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	96 local stream = new_stream(stream_session, stream_callbacks);
2675 101a2a0b8b33 mod_storage_xmlarchive: Pass username to dates Kim Alvefur <zash@zash.se> parents: 2663 diff changeset	97 local dates = self:dates(username) or empty;
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	98 local function reset_stream()
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	99 stream:reset();
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	100 stream_session.notopen = true;
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	101 stream:feed(st.stanza("stream", { xmlns = "jabber:client" }):top_tag());
2602 324a6a3b730b mod_storage_xmlarchive: Rename variable, we can afford to use the full word 'session' Kim Alvefur <zash@zash.se> parents: 2601 diff changeset	102 stream_session.notopen = nil;
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	103 end
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	104 reset_stream();
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	105
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	106 local limit = query.limit;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	107 local start_day, step, last_day = 1, 1, #dates;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	108 local count = 0;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	109 local rev = query.reverse;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	110 if query.start then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	111 local d = dt.date(query.start);
3587 c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	112 for i = start_day, last_day, step do
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	113 if dates[i] < d then
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	114 start_day = i + 1;
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	115 elseif dates[i] >= d then
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	116 start_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	117 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	118 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	119 end
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	120 if query["end"] then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	121 local d = dt.date(query["end"]);
3587 c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	122 for i = last_day, start_day, -step do
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	123 if dates[i] > d then
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	124 last_day = i - 1;
c24d43ababc6 mod_storage_xmlarchive: Try harder to limit range of time to check Kim Alvefur <zash@zash.se> parents: 3585 diff changeset	125 elseif dates[i] <= d then
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	126 last_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	127 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	128 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	129 end
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	130 local items;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	131 local first_item, last_item;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	132 if rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	133 start_day, step, last_day = last_day, -step, start_day;
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	134 if query.before then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	135 local before_day, before_item, items_ = self:_get_idx(username, query.before, dates);
3572 7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	136 if not before_day then
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	137 return nil, "item-not-found";
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	138 elseif before_day <= start_day then
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	139 if before_item then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	140 first_item = before_item - 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	141 else
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	142 first_item = #items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	143 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	144 last_item = 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	145 start_day = before_day;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	146 items = items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	147 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	148 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	149 elseif query.after then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	150 local after_day, after_item, items_ = self:_get_idx(username, query.after, dates);
3572 7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	151 if not after_day then
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	152 return nil, "item-not-found";
7700c9537e90 mod_storage_xmlarchive: Return error if range request reference missing items (see #1325) Kim Alvefur <zash@zash.se> parents: 3571 diff changeset	153 elseif after_day >= start_day then
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	154 if after_item then
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	155 first_item = after_item + 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	156 else
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	157 first_item = 1;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	158 end
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	159 last_item = #items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	160 start_day = after_day;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	161 items = items_;
7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	162 end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	163 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	164
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	165 local date_open, xmlfile;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	166 local function read_xml(date, offset, length)
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	167 if xmlfile and date ~= date_open then
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	168 module:log("debug", "Closing XML file for %s", date_open);
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	169 xmlfile:close();
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	170 xmlfile = nil;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	171 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	172 if not xmlfile then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	173 date_open = date;
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	174 local filename = dm.getpath(username .. "@" .. date, self.host, self.store, "xml");
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	175 local ferr;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	176 xmlfile, ferr = io.open(filename);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	177 if not xmlfile then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	178 module:log("error", "Error: %s", ferr);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	179 return nil, ferr;
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	180 end
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	181 module:log("debug", "Opened XML file %s", filename);
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	182 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	183 local pos, err = xmlfile:seek("set", offset);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	184 if pos ~= offset then
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	185 return nil, err or "seek-failed";
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	186 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	187 return xmlfile:read(length);
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	188 end
995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	189
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	190 return function ()
2054 b7c528027762 mod_storage_xmlarchive: Only try to close xmlfile if it has been opened (fixes traceback with limit=0) Kim Alvefur <zash@zash.se> parents: 2041 diff changeset	191 if limit and count >= limit then if xmlfile then xmlfile:close() end return; end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	192 for d = start_day, last_day, step do
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	193 local date = dates[d];
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	194 if not items then
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	195 module:log("debug", "Loading index for %s", date);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	196 start_day = d;
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	197 items = dm.list_load(username .. "@" .. date, self.host, self.store) or empty;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	198 if not rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	199 first_item, last_item = 1, #items;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	200 else
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	201 first_item, last_item = #items, 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	202 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	203 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	204
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	205 local q_with, q_start, q_end = query.with, query.start, query["end"];
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	206 for i = first_item, last_item, step do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	207 local item = items[i];
2396 544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields Kim Alvefur <zash@zash.se> parents: 2290 diff changeset	208 if not item then
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	209 module:log("warn", "data[%q][%d] is nil", date, i);
2396 544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields Kim Alvefur <zash@zash.se> parents: 2290 diff changeset	210 break;
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields Kim Alvefur <zash@zash.se> parents: 2290 diff changeset	211 end
2634 71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability Kim Alvefur <zash@zash.se> parents: 2633 diff changeset	212
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	213 local i_when, i_with = item.when, item.with;
2634 71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability Kim Alvefur <zash@zash.se> parents: 2633 diff changeset	214
1742 ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	215 if type(i_when) == "string" then
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	216 i_when = dt.parse(i_when);
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	217 end
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	218 if type(i_when) ~= "number" then
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	219 module:log("warn", "data[%q][%d].when is invalid", date, i);
1742 ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	220 break;
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	221 end
2634 71ce798c86cc mod_storage_xmlarchive: Add some spacing to improve readability Kim Alvefur <zash@zash.se> parents: 2633 diff changeset	222
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	223 if (not q_with or i_with == q_with)
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	224 and (not q_start or i_when >= q_start)
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	225 and (not q_end or i_when <= q_end) then
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	226 count = count + 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	227 first_item = i + step;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	228
2633 1330ed88ecd8 mod_storage_xmlarchive: Cache current date in a local Kim Alvefur <zash@zash.se> parents: 2632 diff changeset	229 local data = read_xml(date, item.offset, item.length);
2632 995d4d9f5d89 mod_storage_xmlarchive: Break out XML file reading into a function Kim Alvefur <zash@zash.se> parents: 2602 diff changeset	230 if not data then return end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	231 local ok, err = stream:feed(data);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	232 if not ok then
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	233 module:log("warn", "Parse error in %s@%s/%s/%q[%d]: %s", username, self.host, self.store, i, err);
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	234 reset_stream();
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	235 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	236 if result then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	237 local stanza = result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	238 result = nil;
1752 3f3689a16133 mod_storage_xmlarchive: Return 'when' as number Kim Alvefur <zash@zash.se> parents: 1742 diff changeset	239 return item.id, stanza, i_when, i_with;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	240 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	241 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	242 end
2398 7e922b968b44 mod_storage_xmlarchive: Find item indices for 'before' or 'after' queries and behave as if they were excluded if the items don't exist (thanks MattJ) Kim Alvefur <zash@zash.se> parents: 2397 diff changeset	243 items = nil;
2397 7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	244 if xmlfile then
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	245 xmlfile:close();
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	246 xmlfile = nil;
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed Kim Alvefur <zash@zash.se> parents: 2396 diff changeset	247 end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	248 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	249 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	250 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	251
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	252 function archive:delete(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	253 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	254 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	255 if query.with or query.start or query.after then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	256 return nil, "not-implemented"; -- Only trimming the oldest messages
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	257 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	258 local before = query.before or query["end"] or "9999-12-31";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	259 if type(before) == "number" then before = dt.date(before); else before = before:sub(1, 10); end
2663 6e8c3fc48237 mod_storage_xmlarchive: Use the dates method in more places Kim Alvefur <zash@zash.se> parents: 2634 diff changeset	260 local dates, err = self:dates(username);
2040 459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	261 if not dates or next(dates) == nil then
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	262 if not err then return true end -- already empty
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	263 return dates, err;
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	264 end
2041 7c61ab512d0b mod_storage_xmlarchive: Return earlier if attempting to delete messages older than the oldest existing Kim Alvefur <zash@zash.se> parents: 2040 diff changeset	265 if dates[1] > before then return true; end -- Nothing to delete
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	266 local remaining_dates = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	267 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	268 if dates[d] >= before then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	269 table.insert(remaining_dates, dates[d]);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	270 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	271 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	272 table.sort(remaining_dates);
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	273 local ok, err = dm.list_store(username, self.host, self.store, remaining_dates);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	274 if not ok then return ok, err; end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	275 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	276 if dates[d] < before then
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	277 os.remove(dm.getpath(username .. "@" .. dates[d], self.host, self.store, "list"));
e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	278 os.remove(dm.getpath(username .. "@" .. dates[d], self.host, self.store, "xml"));
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	279 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	280 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	281 return true;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	282 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	283
1831 004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	284 function archive:dates(username)
2917 cd5e6534b813 mod_storage_xmlarchive: Add additional debug logging Kim Alvefur <zash@zash.se> parents: 2815 diff changeset	285 module:log("debug", "Loading root index for %s", username);
2814 1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	286 local dates, err = dm.list_load(username, self.host, self.store);
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	287 if not dates then return dates, err; end
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	288 assert(type(dates[1]) == "string" and type(dates[#dates]) == "string",
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	289 "Archive does not appear to be in xmlarchive format");
1ffbd73c54ba mod_storage_xmlarchive: Add a sanity check to prevent reading from internal storage archives Kim Alvefur <zash@zash.se> parents: 2813 diff changeset	290 return dates;
1831 004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	291 end
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	292
3585 ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	293 function archive:users()
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	294 return dm.users(module.host, self.store, "list");
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	295 end
ddf109d58eff mod_storage_xmlarchive: Add support for user iteration API Kim Alvefur <zash@zash.se> parents: 3572 diff changeset	296
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	297 local provider = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	298 function provider:open(store, typ)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	299 if typ ~= "archive" then return nil, "unsupported-store"; end
2813 e5ce64aee4ac mod_storage_xmlarchive: Pass the hostname into the archive object to make it more self-contained Kim Alvefur <zash@zash.se> parents: 2676 diff changeset	300 return setmetatable({ host = module.host, store = store }, archive_mt);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	301 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	302
1819 1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	303 function provider:purge(username)
2676 1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	304 local encoded_username = dm.path_encode((username or "@") .. "@");
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	305 local basepath = prosody.paths.data .. "/" .. dm.path_encode(module.host);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	306 for store in lfs.dir(basepath) do
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	307 store = basepath .. "/" .. dm.path_encode(store);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	308 if lfs.attributes(store, "mode") == "directory" then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	309 for file in lfs.dir(store) do
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	310 if file:sub(1, #encoded_username) == encoded_username then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	311 if file:sub(-4) == ".xml" or file:sub(-5) == ".list" then
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	312 os.remove(store .. "/" .. file);
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	313 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	314 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	315 end
1b081c8fc1d9 mod_storage_xmlarchive: Attempt to clean up orphaned files despite util.datamanager or internal storage having deleted the date index file (fixes #725) Kim Alvefur <zash@zash.se> parents: 2675 diff changeset	316 return true;
1819 1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	317 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	318 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	319 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	320
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	321 module:provides("storage", provider);
2815 d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	322
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	323
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	324 function module.command(arg)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	325 local jid = require "util.jid";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	326 if arg[1] == "convert" and (arg[2] == "to" or arg[2] == "from") and arg[4] then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	327 local convert;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	328 if arg[2] == "to" then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	329 local xml = require "util.xml";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	330 function convert(user, host, store)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	331 local dates, err = archive.dates({ host = host, store = store }, user);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	332 if not dates then assert(not err, err); return end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	333 assert(dm.list_store(user, host, store, nil));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	334 for _, date in ipairs(dates) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	335 print(date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	336 local items = assert(dm.list_load(user .. "@" .. date, host, store));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	337 local xmlfile = assert(io.open(dm.getpath(user .. "@" .. date, host, store, "xml")));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	338 for _, item in ipairs(items) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	339 assert(xmlfile:seek("set", item.offset));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	340 local data = assert(xmlfile:read(item.length));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	341 assert(#data == item.length, "short read");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	342 data = assert(xml.parse(data));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	343 data = st.preserialize(data);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	344 data.key = item.id;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	345 data.with = item.with;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	346 data.when = tonumber(item.when) or dt.parse(item.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	347 data.attr.stamp = item.when;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	348 data.attr.stamp_legacy = dt.legacy(data.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	349 assert(dm.list_append(user, host, store, data));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	350 end
3491 4e9d4b07e3e9 mod_storage_xmlarchive: Remove per day files during migration Kim Alvefur <zash@zash.se> parents: 3462 diff changeset	351 assert(os.remove(dm.getpath(user .. "@" .. date, host, store, "list")));
4e9d4b07e3e9 mod_storage_xmlarchive: Remove per day files during migration Kim Alvefur <zash@zash.se> parents: 3462 diff changeset	352 assert(os.remove(dm.getpath(user .. "@" .. date, host, store, "xml")));
2815 d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	353 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	354 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	355 else -- convert from internal
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	356 function convert(user, host, store)
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	357 local items, err = dm.list_load(user, host, store);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	358 if not items then assert(not err, err); return end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	359 local dates = {};
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	360 local dayitems, date, xmlfile;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	361 for _, item in ipairs(items) do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	362 local meta = {
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	363 id = item.key;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	364 with = item.with;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	365 when = item.when or dt.parse(item.attr.stamp);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	366 };
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	367 local current_date = dt.date(meta.when);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	368 if current_date ~= date then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	369 if xmlfile then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	370 assert(xmlfile:close());
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	371 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	372 if dayitems then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	373 assert(dm.list_store(user .. "@" .. date, host, store, dayitems));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	374 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	375 print(current_date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	376 dayitems = {};
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	377 date = current_date;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	378 table.insert(dates, date);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	379 xmlfile = assert(io.open(dm.getpath(user .. "@" .. date, host, store, "xml"), "w"));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	380 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	381 item.attr.stamp, item.attr.stamp_legacy = nil, nil;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	382 local stanza = tostring(st.deserialize(item)) .. "\n";
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	383 meta.offset, meta.length = xmlfile:seek(), #stanza;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	384 assert(xmlfile:write(stanza));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	385 table.insert(dayitems, meta);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	386 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	387 assert(xmlfile:close());
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	388 assert(dm.list_store(user .. "@" .. date, host, store, dayitems));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	389 assert(dm.list_store(user, host, store, dates));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	390 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	391 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	392
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	393 local store = arg[4];
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	394 if arg[3] == "internal" then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	395 for i = 5, #arg do
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	396 local user, host = jid.prepped_split(arg[i]);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	397 if not user then
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	398 print(string.format("Argument #%d (%q) is an invalid JID, aborting", i, arg[i]));
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	399 os.exit(1);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	400 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	401 convert(user, host, store);
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	402 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	403 print("Done");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	404 return 0;
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	405 else
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	406 print("Currently only conversion to/from mod_storage_internal is supported");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	407 print("Check out https://modules.prosody.im/mod_migrate");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	408 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	409 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	410 print("prosodyctl mod_storage_xmlarchive convert (from\|to) internal (archive\|archive2\|muc_log) user@host");
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	411 end
d48d4d9ccae7 mod_storage_xmlarchive: Add a prosodyctl command for migrating to/from the internal storage format Kim Alvefur <zash@zash.se> parents: 2814 diff changeset	412

Mercurial > prosody-modules

annotate mod_storage_xmlarchive/mod_storage_xmlarchive.lua @ 3665:b105c4425359