prosody-modules: mod_storage_xmlarchive/mod_storage

annotate mod_storage_xmlarchive/mod_storage_xmlarchive.lua @ 2387:5bfb2ccab2ab

mod_firewall: README: Add more info about specifying rule set files

author	Matthew Wild <mwild1@gmail.com>
date	Sat, 19 Nov 2016 15:59:31 +0000
parents	4786bf0a9334
children	544ee057a23f

rev	line source
1764 a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	1 -- mod_storage_xmlarchive
1765 1a7867eaac9e mod_storage_xmlarchive: Mistake in copypaste of copyright header :P Kim Alvefur <zash@zash.se> parents: 1764 diff changeset	2 -- Copyright (C) 2015 Kim Alvefur
1764 a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	3 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	4 -- This file is MIT/X11 licensed.
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	5 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	6 -- luacheck: ignore unused self
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header Kim Alvefur <zash@zash.se> parents: 1763 diff changeset	7
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	8 local lfs = require "lfs";
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	9 local dm = require "core.storagemanager".olddm;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	10 local hmac_sha256 = require"util.hashes".hmac_sha256;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	11 local st = require"util.stanza";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	12 local dt = require"util.datetime";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	13 local new_stream = require "util.xmppstream".new;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	14 local empty = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	15
2289 aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	16 if not dm.append_raw then
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	17 module:require"datamanager_append_raw";
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	18 end
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility Kim Alvefur <zash@zash.se> parents: 2271 diff changeset	19
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	20 local archive = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	21 local archive_mt = { __index = archive };
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	22
1753 54c8a0cb2996 mod_storage_(archive-capable): Change order of arguments to :append to be the same as return values from :find iterator (see prosody 41725f3df3cc) Kim Alvefur <zash@zash.se> parents: 1752 diff changeset	23 function archive:append(username, _, data, when, with)
54c8a0cb2996 mod_storage_(archive-capable): Change order of arguments to :append to be the same as return values from :find iterator (see prosody 41725f3df3cc) Kim Alvefur <zash@zash.se> parents: 1752 diff changeset	24 if type(when) ~= "number" then
1760 e72f9eac51c8 mod_storage_(various): Order swapping in 54c8a0cb2996 was backwards Kim Alvefur <zash@zash.se> parents: 1759 diff changeset	25 when, with, data = data, when, with;
1753 54c8a0cb2996 mod_storage_(archive-capable): Change order of arguments to :append to be the same as return values from :find iterator (see prosody 41725f3df3cc) Kim Alvefur <zash@zash.se> parents: 1752 diff changeset	26 end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	27 if getmetatable(data) ~= st.stanza_mt then
1739 940a4ab75cec mod_storage_xmlarchive: This module only supports storage of stanzas, log a warning about attempts to do otherwise Kim Alvefur <zash@zash.se> parents: 1738 diff changeset	28 module:log("error", "Attempt to store non-stanza object, traceback: %s", debug.traceback());
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	29 return nil, "unsupported-datatype";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	30 end
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	31
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	32 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	33 data = tostring(data) .. "\n";
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	34
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	35 local day = dt.date(when);
1969 e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	36 local ok, err = dm.append_raw(username.."@"..day, module.host, self.store, "xml", data);
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	37 if not ok then
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	38 return nil, err;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	39 end
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	40
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	41 -- If the day-file is missing then we need to add it to the list of days
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	42 local first_of_day = not lfs.attributes(dm.getpath(username .. "@" .. day, module.host, self.store, "list"));
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	43
1969 e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	44 local offset = ok and err;
1740 11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return' Kim Alvefur <zash@zash.se> parents: 1739 diff changeset	45
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	46 local id = day .. "-" .. hmac_sha256(username.."@"..day.."+"..offset, data, true):sub(-16);
1742 ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	47 ok, err = dm.list_append(username.."@"..day, module.host, self.store, { id = id, when = dt.datetime(when), with = with, offset = offset, length = #data });
2290 4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before Kim Alvefur <zash@zash.se> parents: 2289 diff changeset	48 if ok and first_of_day then
2038 a85b5c3791dc mod_storage_xmlarchive: Collect return value when adding to date index Kim Alvefur <zash@zash.se> parents: 1969 diff changeset	49 ok, err = dm.list_append(username, module.host, self.store, day);
1969 e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	50 end
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	51 if not ok then
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	52 return nil, err;
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here) Kim Alvefur <zash@zash.se> parents: 1831 diff changeset	53 end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	54 return id;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	55 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	56
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	57 function archive:find(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	58 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	59 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	60
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	61 local result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	62 local function cb(_, stanza)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	63 if result then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	64 module:log("warn", "Multiple items in chunk");
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	65 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	66 result = stanza;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	67 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	68
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	69 local stream_sess = { notopen = true };
2183 13a8bbf256dd mod_storage_xmlarchive: Pass a default namespace to xmppstream so result items default to xmlns = nil (fixes non-mam use, like mod_mam_muc sending history) Kim Alvefur <zash@zash.se> parents: 2054 diff changeset	70 local stream = new_stream(stream_sess, { handlestanza = cb, stream_ns = "jabber:client", default_ns = "jabber:client" });
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	71 local dates = dm.list_load(username, module.host, self.store) or empty;
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	72 local function reset_stream()
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	73 stream:reset();
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	74 stream_sess.notopen = true;
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	75 stream:feed(st.stanza("stream", { xmlns = "jabber:client" }):top_tag());
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	76 stream_sess.notopen = nil;
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	77 end
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	78 reset_stream();
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	79
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	80 local limit = query.limit;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	81 local start_day, step, last_day = 1, 1, #dates;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	82 local count = 0;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	83 local rev = query.reverse;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	84 local in_range = not (query.after or query.before);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	85 if query.after or query.start then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	86 local d = query.after and query.after:sub(1, 10) or dt.date(query.start);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	87 for i = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	88 if dates[i] == d then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	89 start_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	90 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	91 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	92 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	93 if query.before or query["end"] then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	94 local d = query.before and query.before:sub(1, 10) or dt.date(query["end"]);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	95 for i = #dates, 1, -1 do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	96 if dates[i] == d then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	97 last_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	98 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	99 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	100 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	101 if rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	102 start_day, step, last_day = last_day, -step, start_day;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	103 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	104 local items, xmlfile;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	105 local first_item, last_item;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	106
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	107 return function ()
2054 b7c528027762 mod_storage_xmlarchive: Only try to close xmlfile if it has been opened (fixes traceback with limit=0) Kim Alvefur <zash@zash.se> parents: 2041 diff changeset	108 if limit and count >= limit then if xmlfile then xmlfile:close() end return; end
1726 160c35d2a5a2 mod_storage_xmlarchive: Improve logging of parse errors Kim Alvefur <zash@zash.se> parents: 1690 diff changeset	109 local filename;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	110
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	111 for d = start_day, last_day, step do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	112 if d ~= start_day or not items then
1728 a51beeb5aecf mod_storage_xmlarchive: Adjust wording in log message Kim Alvefur <zash@zash.se> parents: 1727 diff changeset	113 module:log("debug", "Loading items from %s", dates[d]);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	114 start_day = d;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	115 items = dm.list_load(username .. "@" .. dates[d], module.host, self.store) or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	116 if not rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	117 first_item, last_item = 1, #items;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	118 else
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	119 first_item, last_item = #items, 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	120 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	121 local ferr;
1726 160c35d2a5a2 mod_storage_xmlarchive: Improve logging of parse errors Kim Alvefur <zash@zash.se> parents: 1690 diff changeset	122 filename = dm.getpath(username .. "@" .. dates[d], module.host, self.store, "xml");
160c35d2a5a2 mod_storage_xmlarchive: Improve logging of parse errors Kim Alvefur <zash@zash.se> parents: 1690 diff changeset	123 xmlfile, ferr = io.open(filename);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	124 if not xmlfile then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	125 module:log("error", "Error: %s", ferr);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	126 return;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	127 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	128 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	129
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	130 local q_with, q_start, q_end = query.with, query.start, query["end"];
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	131 for i = first_item, last_item, step do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	132 local item = items[i];
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	133 local i_when, i_with = item.when, item.with;
1742 ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	134 if type(i_when) == "string" then
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	135 i_when = dt.parse(i_when);
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	136 end
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	137 if type(i_when) ~= "number" then
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	138 module:log("warn", "data[%q][%d].when is invalid", dates[d], i);
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	139 break;
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua Kim Alvefur <zash@zash.se> parents: 1741 diff changeset	140 end
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	141 if not item then
1730 e02af2b32f30 mod_storage_xmlarchive: Increase severity of log message regarding unexpected missing items Kim Alvefur <zash@zash.se> parents: 1729 diff changeset	142 module:log("warn", "data[%q][%d] is nil", dates[d], i);
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	143 break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	144 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	145 if xmlfile and in_range
1741 07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	146 and (not q_with or i_with == q_with)
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	147 and (not q_start or i_when >= q_start)
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop Kim Alvefur <zash@zash.se> parents: 1740 diff changeset	148 and (not q_end or i_when <= q_end) then
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	149 count = count + 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	150 first_item = i + step;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	151
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	152 xmlfile:seek("set", item.offset);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	153 local data = xmlfile:read(item.length);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	154 local ok, err = stream:feed(data);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	155 if not ok then
1726 160c35d2a5a2 mod_storage_xmlarchive: Improve logging of parse errors Kim Alvefur <zash@zash.se> parents: 1690 diff changeset	156 module:log("warn", "Parse error in %s at %d+%d: %s", filename, item.offset, item.length, err);
1727 8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures Kim Alvefur <zash@zash.se> parents: 1726 diff changeset	157 reset_stream();
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	158 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	159 if result then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	160 local stanza = result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	161 result = nil;
1752 3f3689a16133 mod_storage_xmlarchive: Return 'when' as number Kim Alvefur <zash@zash.se> parents: 1742 diff changeset	162 return item.id, stanza, i_when, i_with;
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	163 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	164 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	165 if (rev and item.id == query.after) or
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	166 (not rev and item.id == query.before) then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	167 in_range = false;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	168 limit = count;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	169 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	170 if (rev and item.id == query.before) or
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	171 (not rev and item.id == query.after) then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	172 in_range = true;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	173 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	174 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	175 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	176 if xmlfile then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	177 xmlfile:close();
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	178 xmlfile = nil;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	179 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	180 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	181 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	182
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	183 function archive:delete(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	184 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	185 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	186 if query.with or query.start or query.after then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	187 return nil, "not-implemented"; -- Only trimming the oldest messages
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	188 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	189 local before = query.before or query["end"] or "9999-12-31";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	190 if type(before) == "number" then before = dt.date(before); else before = before:sub(1, 10); end
2040 459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	191 local dates, err = dm.list_load(username, module.host, self.store);
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	192 if not dates or next(dates) == nil then
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	193 if not err then return true end -- already empty
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	194 return dates, err;
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive Kim Alvefur <zash@zash.se> parents: 2038 diff changeset	195 end
2041 7c61ab512d0b mod_storage_xmlarchive: Return earlier if attempting to delete messages older than the oldest existing Kim Alvefur <zash@zash.se> parents: 2040 diff changeset	196 if dates[1] > before then return true; end -- Nothing to delete
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	197 local remaining_dates = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	198 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	199 if dates[d] >= before then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	200 table.insert(remaining_dates, dates[d]);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	201 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	202 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	203 table.sort(remaining_dates);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	204 local ok, err = dm.list_store(username, module.host, self.store, remaining_dates);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	205 if not ok then return ok, err; end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	206 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	207 if dates[d] < before then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	208 os.remove(dm.getpath(username .. "@" .. dates[d], module.host, self.store, "list"));
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	209 os.remove(dm.getpath(username .. "@" .. dates[d], module.host, self.store, "xml"));
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	210 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	211 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	212 return true;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	213 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	214
1831 004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	215 function archive:dates(username)
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	216 return dm.list_load(username, module.host, self.store);
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	217 end
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages Kim Alvefur <zash@zash.se> parents: 1819 diff changeset	218
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	219 local provider = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	220 function provider:open(store, typ)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	221 if typ ~= "archive" then return nil, "unsupported-store"; end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	222 return setmetatable({ store = store }, archive_mt);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	223 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	224
1819 1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	225 function provider:purge(username)
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	226 for store in dm.stores(username, module.host) do
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	227 local dates = dm.list_load(username, module.host, store) or empty;
2271 b34f4591366a mod_storage_xmlarchive: Fix date pattern in purge (related to #725) Kim Alvefur <zash@zash.se> parents: 2183 diff changeset	228 if dates[1] and type(dates[1]) == "string" and dates[1]:match("^%d%d%d%d%-%d%d%-%d%d$") then
1819 1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	229 module:log("info", "Store %s looks like an archive store, emptying it...", store);
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	230 provider:open(store, "archive"):delete(username);
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	231 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	232 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	233 return true;
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	234 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users) Kim Alvefur <zash@zash.se> parents: 1793 diff changeset	235
1690 8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files Kim Alvefur <zash@zash.se> parents: diff changeset	236 module:provides("storage", provider);

Mercurial > prosody-modules

annotate mod_storage_xmlarchive/mod_storage_xmlarchive.lua @ 2387:5bfb2ccab2ab