Mercurial > prosody-modules

changeset 1777:c353acd1d366

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merge with Goffi
author Kim Alvefur <zash@zash.se>
date Mon, 10 Aug 2015 21:13:31 +0200
parents fb2b9a2e2316 (diff) e7b5ab44339c (current diff)
children 32604bf33a4c
files
diffstat 23 files changed, 730 insertions(+), 114 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_adhoc_blacklist/mod_adhoc_blacklist.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -0,0 +1,90 @@
+-- mod_adhoc_blacklist
+--
+-- http://xmpp.org/extensions/xep-0133.html#edit-blacklist
+--
+-- Copyright (C) 2015 Kim Alvefur
+--
+-- This file is MIT/X11 licensed.
+--
+
+module:depends("adhoc");
+local adhoc = module:require "adhoc";
+local st = require"util.stanza";
+local set = require"util.set";
+local dataform = require"util.dataforms";
+local adhoc_inital_data = require "util.adhoc".new_initial_data_form;
+
+local blocklist_form = dataform.new {
+	title = "Editing the Blacklist";
+	instructions = "Fill out this form to edit the list of entities with whom communications are disallowed.";
+	{
+		type = "hidden";
+		name = "FORM_TYPE";
+		value = "http://jabber.org/protocol/admin";
+	};
+	{
+		type = "jid-multi";
+		name = "blacklistjids";
+		label = "The blacklist";
+	};
+}
+
+local blocklists = module:open_store("blocklist");
+
+local blocklist_handler = adhoc_inital_data(blocklist_form, function ()
+	local blacklistjids = {};
+	local blacklist = blocklists:get();
+	if blacklist then
+		for jid in pairs(blacklist) do
+			table.insert(blacklistjids, jid);
+		end
+	end
+	return { blacklistjids = blacklistjids };
+end, function(fields, form_err)
+	if form_err then
+		return { status = "completed", error = { message = "Problem in submitted form" } };
+	end
+	local blacklistjids = set.new(fields.blacklistjids);
+	local ok, err = blocklists:set(nil, blacklistjids._items);
+	if ok then
+		return { status = "completed", info = "Blacklist updated" };
+	else
+		return { status = "completed", error = { message = "Error saving blacklist: "..err } };
+	end
+end);
+
+module:add_item("adhoc", adhoc.new("Edit Blacklist", "http://jabber.org/protocol/admin#edit-blacklist", blocklist_handler, "admin"));
+
+local function is_blocked(host)
+	local blacklistjids = blocklists:get();
+	return blacklistjids and blacklistjids[host];
+end
+
+module:hook("route/remote", function (event)
+	local origin, stanza = event.origin, event.stanza;
+	if is_blocked(event.to_host) then
+		if origin and stanza then
+			origin.send(st.error_reply(stanza, "cancel", "not-allowed", "Communication with this domain is not allowed"));
+			return true;
+		end
+		return false;
+	end
+end, 1000);
+
+
+module:hook("s2s-stream-features", function (event)
+	local session = event.origin;
+	if is_blocked(session.from_host) then
+		session:close("policy-violation");
+		return false;
+	end
+end, 1000);
+
+module:hook("stanza/http://etherx.jabber.org/streams:features", function (event)
+	local session = event.origin;
+	if is_blocked(session.to_host) then
+		session:close("policy-violation");
+		return true;
+	end
+end, 1000);
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_admin_blocklist/mod_admin_blocklist.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -0,0 +1,59 @@
+-- mod_admin_blocklist
+--
+-- If a local admin has blocked a domain, don't allow s2s to that domain
+--
+-- Copyright (C) 2015 Kim Alvefur
+--
+-- This file is MIT/X11 licensed.
+--
+
+module:depends("blocklist");
+
+local st = require"util.stanza";
+local jid_split = require"util.jid".split;
+
+local admins = module:get_option_inherited_set("admins", {}) /
+	function (admin) -- Filter out non-local admins
+		local user, host = jid_split(admin);
+		if host == module.host then return user; end
+	end
+
+local blocklists = module:open_store("blocklist");
+
+local function is_blocked(host)
+	for admin in admins do
+		local blocklist = blocklists:get(admin);
+		if blocklist and blocklist[host] then
+			return true;
+		end
+	end
+end
+
+module:hook("route/remote", function (event)
+	local origin, stanza = event.origin, event.stanza;
+	if is_blocked(event.to_host) then
+		if origin and stanza then
+			origin.send(st.error_reply(stanza, "cancel", "not-allowed", "Communication with this domain is not allowed"));
+			return true;
+		end
+		return false;
+	end
+end, 1000);
+
+
+module:hook("s2s-stream-features", function (event)
+	local session = event.origin;
+	if is_blocked(session.from_host) then
+		session:close("policy-violation");
+		return false;
+	end
+end, 1000);
+
+module:hook("stanza/http://etherx.jabber.org/streams:features", function (event)
+	local session = event.origin;
+	if is_blocked(session.to_host) then
+		session:close("policy-violation");
+		return true;
+	end
+end, 1000);
+
--- a/mod_auth_http_async/mod_auth_http_async.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_auth_http_async/mod_auth_http_async.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -7,7 +7,6 @@
 -- COPYING file in the source package for more information.
 --
 
-local usermanager = require "core.usermanager";
 local new_sasl = require "util.sasl".new;
 local base64 = require "util.encodings".base64.encode;
 local waiter =require "util.async".waiter;
@@ -66,7 +65,7 @@
 function provider.get_sasl_handler()
 	return new_sasl(host, {
 		plain_test = function(sasl, username, password, realm)
-			return usermanager.test_password(username, realm, password), true;
+			return provider.test_password(username, realm, password), true;
 		end
 	});
 end
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_benchmark_storage/mod_benchmark_storage.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -0,0 +1,119 @@
+-- mod_benchmark_storage
+-- Copyright (C) 2015 Kim Alvefur
+--
+-- Prime numbers are pretty cool
+
+local gettime = require"socket".gettime;
+
+local sm = require"core.storagemanager";
+local um = require"core.usermanager";
+local mm = require"core.modulemanager";
+
+local test_data, test_users;
+
+function module.command(arg)
+	local test_driver = arg[1];
+	if not test_driver then
+		return print("Usage: prosodyctl mod_"..module.name.." <storage driver>");
+	end
+
+	sm.initialize_host("localhost");
+	um.initialize_host("localhost");
+
+	local start_time = gettime();
+	local storage = assert(sm.load_driver("localhost", test_driver));
+	storage = assert(storage:open("benchmark"));
+	-- for i = 1, 23 do
+		-- storage:set(test_users[i], test_data);
+	-- end
+	local floor, sin, random, pi = math.floor, math.sin, math.random, math.pi;
+	for i = 1, 10079 do
+		if i % 11 == 1 then
+			storage:set(test_users[i%23+1], test_data[3-floor(sin(random()*pi)*3)]);
+		else
+			storage:get(test_users[i%23+1]);
+		end
+		if i % 151 == 0 then
+			-- Give indication of progress
+			io.write("*");
+			io.flush();
+		end
+	end
+	-- Cleanup
+	for i = 1, 23 do
+		storage:set(test_users[i], nil);
+	end
+	mm.unload("localhost", "storage_"..test_driver);
+	local time_taken = gettime() - start_time;
+	io.write("\27[0G\27[K"); -- Clear current line
+	io.flush();
+	print(("Took %fs with mod_storage_%s"):format(time_taken, test_driver));
+end
+
+-- 23 usernames
+test_users = {
+	"tritonymph"; "ankylotomy"; "tron"; "barbaric"; "twiddler"; 
+	"spiritful"; "unmollifiably"; "suggestion"; "presubsistence"; 
+	"unneeded"; "taxemic"; "teloteropathic"; "nonbending"; "mev"; 
+	"septifragally"; "clame"; "obsolescent"; "unconceivable"; 
+	"foolishly"; "conjunctur"; "precirculation"; "bethump"; "vermivorous";
+};
+
+test_data = {
+	{ some_data = "tiny data" };
+	--
+	{ [false] = { version = 1; pending = {}; }; -- Medium data
+		["user@example.com"] = { subscription = "both"; groups = {};
+			name = "My Best Friend"; }; };
+	--
+	{ attr = { xmlns = "vcard-temp"; }; name = "vCard"; -- The largest data
+	{ attr = { xmlns = "vcard-temp"; }; name = "NICKNAME"; "Buster"; };
+	{ attr = { xmlns = "vcard-temp"; }; name = "PHOTO";
+	{ attr = { xmlns = "vcard-temp"; }; name = "TYPE"; "image/jpeg"; };
+	{ attr = { xmlns = "vcard-temp"; }; name = "BINVAL"; [[
+	/9j/4AAQSkZJRgABAgAAZABkAAD/2wBDAAgFBQUGBQgGBggLBwYHCw0JCAgJDQ8MDA0MDA8RDAwM
+	DAwMEQ4RERIREQ4XFxgYFxcgICAgICQkJCQkJCQkJCT/2wBDAQgICA8ODxwTExwfGRQZHyQkJCQk
+	JCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCT/wAARCAA8ADwDAREA
+	AhEBAxEB/8QAGwAAAgMBAQEAAAAAAAAAAAAABgcDBAUCCAH/xAA9EAACAQIEAgcECAMJAAAAAAAB
+	AgMEEQAFEhMGIQcUFSIxQWEWQlFSIyQyM2JxgZElcqEXJjSCksHR8PH/xAAbAQACAwEBAQAAAAAA
+	AAAAAAAEBQECAwYAB//EADcRAAEDAwIDBAcGBwAAAAAAAAEAAgMEESESMQUTQSJRYYEUIzJCcaGx
+	BhUWkdHhQ1JyguLw8f/aAAwDAQACEQMRAD8ASI+7+N8WVuiNuiPgaHinPXkr115TloWWpjvbedj9
+	FCT8psS3py88I+N8SNPHZvtu28PFFU0Oo36L0dDTxQxrHGqpEgCoigBVUCwVQPADHz3ckndMLr7M
+	jFOSsR5BL3v5eF8edfyUtOVVENZJGdcDR8ySTc8v1APP8sYPjwStg9oO4Q/xTw9Q5vl0tFXw7tPI
+	De47ynyeNvJlPhgrhde6CQOBROlkjdLsrzdnuUVGS5rUZdMdbQN3HHLWh7ySf5hj6pTzCWMOHVc3
+	U05ikLCtn2Io/Z/tb2goetbe92bt1O9f5NW3bV6/Z9cNfQJe76IH0lurT1Q2SAAPMW/fABRCe3Qx
+	VZTkHR1NnOZTLBHU1U88jEd5khAiXSPMDSccPx1jp6oMbkiw/PKa0rDowF1WdN1bXTCDhnLN0yoX
+	p5ahlJYKdLu8at9GB64Y0/2TcGa538pned/7R1+irzoydLQZHnoNvNRTZX0wZs9LHX522XmoLt9T
+	coAttY1CFUChF5ePPDWGHhkJNo3yfGwVHU8zhu1nzWPmfB3FsGtn4nlnZH0d+pqFu37thjFW8L2d
+	TWHkVmeHVNrtkCxIuMOKOF8x6pm01RW09jqp3nOvTf7cU3zC3g3LFa/g/DauAupWiOUdLdPgoiqq
+	mnktN2gUW5VT8L9IeTisraI9YS8bM9hUR3J0MtRGAWB9bj0xw9VUT8PkDWnsnPh44KdiOOoZqIsV
+	H7B/xzsrqw7F7H6v2htpu9Y3b7mrw3f08MdP+MYvRuZ/GvbR0+P9KTfc/rbe7/N1S94A4Upc4qJc
+	wzQfwjLyokiuVM8zi6QBhzC6RqcjnbkPHE8TrHQtsz23fLx/RTQUnNdn2QmHxBmdJSw9dEKTjJaM
+	V9PlunRTjVMtHTRugt3AXJCjCOkhc+9zhxsT73endTM2Fmlu9v2WnwzlVJQrvbcSq4NTU7UQTvzW
+	dFVie5Gt7KnwwyqaySZ3bJdpwL9yygp2xts0Wvv8UV5jmQy+lgaaHflq20qVIVIkK+99oryGLOYA
+	FlHcu+CG6+uylsygo2BV6lC1OyhQZGI0hQ7EBrfDA7Wbohzu9LvjzKnjqDVTqeqzsqxsCCUdVb7X
+	vDVYXwbTPLSC3BQk7bjOy2OhbOqOKnqcvkUpNNMDAefejIY6fTSfP1wg+0tM95D77Bb0J7Fk1ttd
+	Gry8NXnji7m9kTqzZJ/oyhgm4fenYrv0dXJJIoN+7KiaHP8Aotj6FxcHnA9C36IbhQAYR1uiLP8A
+	Kes09VVkkQHL5oKgAXcbbrVQuL8vvIrH88C0lSGENPU4+OyIqog4X7ltUCRPC7ogfbQM8Xk1vPu+
+	PLwwYQQVAVyWET00FNX6o1XvtMshu4t9jkb4uXEhZWAKAeO4+J1qsqroI1jyennLUYbuyGfUBNqa
+	2pbe76LgiHSGm9wSsJy4vxYhUukfMkMSQ1ZEk4QmBxbS5va/L4c+eLU7SThRUOACFcizJsvzGmrI
+	320hdST4DTf/AIxaqh1tLd16B+mxKOv7Wq7tbrOz/dnb2tuw39Wr/Ffv7vy+uFv4Vj9Hvq9be/8A
+	j+XXvWP3l6/bsXt+6EujGvShzmvrqmfaoaeglecM+hJJSQlLGxseZlbl+uH89G2dug+XgUJBVGN5
+	cPNM3hXijhXO/qy1iR1UwKSUVQQshZu6Ql+7IOdhpxxXFeH1ULhZhIHvNz/xOo66OQYPkVUycV2X
+	b2W1p2amgdo5ApIYpq0o128mW2HQkbI0OHX5KGHFlvFlo6VElmeZRJs07HuoEc8t1m8VXnfzONGG
+	4K84ZCq8SxVJlaQSUppRCqQrI5GmUkWEZJa48efni7g3os2uN8pa9IUqyUdEyrZigEiGxMTrzZBb
+	/pwVSElxCHqh2boUjnhnpmgiRmmCXkcgWVQRaw/Owwy5QAuN0tM5dvsp+v0/Z+ru7ttvZ1jXq1ad
+	O14288a2wsrqLPKCfKn7NDnZ1a3Ye/KnK7eig90Y20aQFlqvcKjHLIJNcTFZVIkjYeKuDcMPUHFg
+	dwFByE/qrN8v4ioEz+gbXVMIap0FwplmiiWupVY/JIobl4N+eAK2mBjvbI/3zR1FOQ8C+DhQHiPI
+	aQOssgklj70lMQ17Hlq0WZmJIwh0OOU7Lhm6EuOeIcizuJJjRyRadRWfQ0NlB069txpbkLchywXC
+	Xg4shJtJGboPXMUrqzQokmpaaJ2LSNbmsZRZG8Od8MaKHt3KAqpbtsLrKy/UpdlJ740HSbGwZSPH
+	8QGGDWAtv3lBA2KKOq5R7H9V00/a3aHZe7Ybu1q67v6v5vor/LimnOi3vfJX1Y1eF1qcTcMVWdTu
+	Mqp5KqS4cGEalWS3NWfkouPXGr5GgZKzDCThQ5J0KcYVFZEayNKLLzpMs+oO4QgkiKLlqceHPlgR
+	9cGG7QSUTHSFxuSAEzJOG5svyJMiyWjqI48vjaahqHkR92WRvpon1FbO9tXwGFj5ZnG56ppHHC0W
+	wh7hzgfj6nk6zWwxrK8gLJJUoymJjeUMoDd/4G/jiksOqw2XoZw0G+Vu1XAtZNUsyU1DHSSRhZIp
+	JJJZNY99e6EA/DjI0thg5WjasXyMIO4n6MeJp2bs9KRouYKrJtWJFvAp/vguik5TruyhascwdnGU
+	LwcB8Y5ejCoyiWUchqj0TAWcNrXaZz5fDDZtVGWAXF/1S407wdlW9m8+7T6x1Gv1bm7t9Xkte9/l
+	+OCdcHN16wsOXLy9NvBOCo9tesL2L1Ps+30PVvufDuatvnb+mF8mr3UXHy/e+eyYUduqQ733mga/
+	5rDVb9cCZstha+FC+1c2v62xGFJXA2vO/pbFQpXMmzbz8fPwxBspCoVWzq5f0xkVqFnyaNR29Wr8
+	FsQtGrr65p97R/X/ANxCn6r/2Q==]]; }; }; };
+};
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_compact_resource/mod_compact_resource.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -0,0 +1,12 @@
+
+local base64_encode = require"util.encodings".base64.encode;
+local random_bytes = require"util.random".bytes;
+
+local b64url = { ["+"] = "-", ["/"] = "_", ["="] = "" };
+local function random_resource()
+	return base64_encode(random_bytes(8)):gsub("[+/=]", b64url);
+end
+
+module:hook("pre-resource-bind", function (event)
+	event.resource = random_resource();
+end);
--- a/mod_filter_chatstates/mod_filter_chatstates.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_filter_chatstates/mod_filter_chatstates.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -1,9 +1,6 @@
 local filters = require "util.filters";
 local st = require "util.stanza";
 
-local dummy_stanza_mt = setmetatable({ __tostring = function () return ""; end }, { __index = st.stanza_mt });
-local dummy_stanza = setmetatable(st.stanza(), dummy_stanza_mt);
-
 module:depends("csi");
 
 local function filter_chatstates(stanza)
@@ -11,11 +8,11 @@
 		stanza = st.clone(stanza);
 		stanza:maptags(function (tag)
 			if tag.attr.xmlns ~= "http://jabber.org/protocol/chatstates" then
-				return tag;
+				return tag
 			end
 		end);
 		if #stanza.tags == 0 then
-			return dummy_stanza;
+			return nil;
 		end
 	end
 	return stanza;
--- a/mod_http_muc_log/http_muc_log.html	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_http_muc_log/http_muc_log.html	Mon Aug 10 21:13:31 2015 +0200
@@ -44,7 +44,7 @@
 <header>
 <h1 title="xmpp:{jid?}">{title}</h1>
 <nav>{links#
-<a class="{rel?}" href="{href}" rel="{rel?}">{text}</a>}
+<a class="{item.rel?}" href="{item.href}" rel="{item.rel?}">{item.text}</a>}
 </nav>
 </header>
 <hr>
@@ -52,33 +52,33 @@
 <nav>
 <dl class="room-list">
 {rooms#
-<dt class="name"><a href="{href}">{name}</a></dt>
-<dd class="description">{description?}</dd>}
+<dt class="name"><a href="{item.href}">{item.name}</a></dt>
+<dd class="description">{item.description?}</dd>}
 </dl>
 {years#
-<h2 id="{year}">{year}</h2>
-{months#
-<table id="{month}-{year}">
-<caption>{month}</caption>
-<tr><th>Mon</th><th>Tue</th><th>Wed</th><th>Thu</th><th>Fri</th><th>Sat</th><th>Sun</th></tr>{weeks#
-<tr>{days#<td>{href&<a href="{href}">}{day?&nbsp;}{href&</a>}</td>}</tr>}
+<h2 id="{item.year}">{item.year}</h2>
+{item.months#
+<table id="{item.month}-{item.year}">
+<caption>{item.month}</caption>
+<tr><th>Mon</th><th>Tue</th><th>Wed</th><th>Thu</th><th>Fri</th><th>Sat</th><th>Sun</th></tr>{item.weeks#
+<tr>{item.days#<td>{item.href&<a href="{item.href}">}{item.day?&nbsp;}{item.href&</a>}</td>}</tr>}
 </table>
 }
 }
 </nav>
 <ol class="chat-logs">{lines#
-<li class="{st_name} {st_type?}" id="{key}">
-<a class="time" href="#{key}"><time datetime="{datetime}">{time}</time></a>
-<b class="nick">{nick}</b>
-<em class="verb">{verb?}</em>
-<q class="body">{body?}</q>
+<li class="{item.st_name} {item.st_type?}" id="{item.key}">
+<a class="time" href="#{item.key}"><time datetime="{item.datetime}">{item.time}</time></a>
+<b class="nick">{item.nick}</b>
+<em class="verb">{item.verb?}</em>
+<q class="body">{item.body?}</q>
 </li>}
 </ol>
 </div>
 <hr>
 <footer>
 <nav>{links#
-<a class="{rel?}" href="{href}" rel="{rel?}">{text}</a>}
+<a class="{item.rel?}" href="{item.href}" rel="{item.rel?}">{item.text}</a>}
 </nav>
 <br>
 <div class="powered-by">Prosody</div>
--- a/mod_http_muc_log/mod_http_muc_log.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_http_muc_log/mod_http_muc_log.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -1,4 +1,3 @@
-local st = require "util.stanza";
 local mt = require"util.multitable";
 local datetime = require"util.datetime";
 local jid_split = require"util.jid".split;
@@ -7,9 +6,8 @@
 local it = require"util.iterators";
 local gettime = require"socket".gettime;
 local url = require"socket.url";
-local xml_escape = st.xml_escape;
-local t_concat = table.concat;
 local os_time, os_date = os.time, os.date;
+local render = require"util.interpolation".new("%b{}", require"util.stanza".xml_escape);
 
 local archive = module:open_store("muc_log", "archive");
 
@@ -33,45 +31,6 @@
 
 module:depends"http";
 
-local function render(template, values)
-	-- This function takes a string template and a table of values.
-	-- Sequences like {name} in the template string are substituted
-	-- with values from the table, optionally depending on a modifier
-	-- symbol.
-	--
-	-- Variants are:
-	-- {name} is substituted for values["name"] and is XML escaped
-	-- {name? sub-template } renders a sub-template if values["name"] is false-ish
-	-- {name& sub-template } renders a sub-template if values["name"] is true-ish
-	-- {name# sub-template } renders a sub-template using an array of values
-	-- {name!} is substituted *without* XML escaping
-	return (template:gsub("%b{}", function (block)
-		local name, opt, e = block:sub(2, -2):match("^([%a_][%w_]*)(%p?)()");
-		local value = values[name];
-		if opt == '#' then
-			if not value or not value[1] then return ""; end
-			local out, subtpl = {}, block:sub(e+1, -2);
-			for i=1, #value do
-				out[i] = render(subtpl, value[i]);
-			end
-			return t_concat(out);
-		elseif opt == '&' then
-			if not value then return ""; end
-			return render(block:sub(e+1, -2), values);
-		elseif opt == '?' and not value then
-			return render(block:sub(e+1, -2), values);
-		elseif value ~= nil then
-			if type(value) ~= "string" then
-				value = tostring(value);
-			end
-			if opt ~= '!' then
-				return xml_escape(value);
-			end
-			return value;
-		end
-	end));
-end
-
 local template;
 do
 	local template_file = module:get_option_string(module.name .. "_template", module.name .. ".html");
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_http_upload/mod_http_upload.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -0,0 +1,110 @@
+-- mod_http_upload
+--
+-- Copyright (C) 2015 Kim Alvefur
+--
+-- This file is MIT/X11 licensed.
+-- 
+-- Implementation of HTTP Upload file transfer mechanism used by Conversations
+--
+
+-- imports
+local st = require"util.stanza";
+local lfs = require"lfs";
+local join_path = require"util.paths".join;
+local uuid = require"util.uuid".generate;
+
+-- depends
+module:depends("http");
+
+-- namespace
+local xmlns_http_upload = "eu:siacs:conversations:http:upload";
+
+module:add_feature(xmlns_http_upload);
+
+-- state
+local pending_slots = module:shared("upload_slots");
+
+local storage_path = join_path(prosody.paths.data, module.name);
+lfs.mkdir(storage_path);
+
+-- hooks
+module:hook("iq/host/"..xmlns_http_upload..":request", function (event)
+	local stanza, origin = event.stanza, event.origin;
+	-- local clients only
+	if origin.type ~= "c2s" then
+		origin.send(st.error_reply(stanza, "cancel", "not-authorized"));
+		return true;
+	end
+	-- validate
+	local filename = stanza.tags[1]:get_child_text("filename");
+	if not filename or filename:find("/") then
+		origin.send(st.error_reply(stanza, "modify", "bad-request"));
+		return true;
+	end
+	local reply = st.reply(stanza);
+	reply:tag("slot", { xmlns = xmlns_http_upload });
+	local random = uuid();
+	pending_slots[random.."/"..filename] = origin.full_jid;
+	local url = module:http_url() .. "/" .. random .. "/" .. filename;
+	reply:tag("get"):text(url):up();
+	reply:tag("put"):text(url):up();
+	origin.send(reply);
+	return true;
+end);
+
+-- http service
+local function upload_data(event, path)
+	if not pending_slots[path] then
+		return 401;
+	end
+	local random, filename = path:match("^([^/]+)/([^/]+)$");
+	if not random then
+		return 400;
+	end
+	local dirname = join_path(storage_path, random);
+	if not lfs.mkdir(dirname) then
+		module:log("error", "Could not create directory %s for upload", dirname);
+		return 500;
+	end
+	local full_filename = join_path(dirname, filename);
+	local fh, ferr = io.open(full_filename, "w");
+	if not fh then
+		module:log("error", "Could not open file %s for upload: %s", full_filename, ferr);
+		return 500;
+	end
+	local ok, err = fh:write(event.request.body);
+	if not ok then
+		module:log("error", "Could not write to file %s for upload: %s", full_filename, err);
+		os.remove(full_filename);
+		return 500;
+	end
+	ok, err = fh:close();
+	if not ok then
+		module:log("error", "Could not write to file %s for upload: %s", full_filename, err);
+		os.remove(full_filename);
+		return 500;
+	end
+	module:log("info", "File uploaded by %s to slot %s", pending_slots[path], random);
+	pending_slots[path] = nil;
+	return 200;
+end
+
+local serve_uploaded_files = module:depends("http_files").serve(storage_path);
+
+local function size_only(request, data)
+	request.headers.content_size = #data;
+	return 200;
+end
+
+local function serve_head(event, path)
+	event.send = size_only;
+	return serve_uploaded_files(event, path);
+end
+
+module:provides("http", {
+	route = {
+		["GET /*"] = serve_uploaded_files;
+		["HEAD /*"] = serve_head;
+		["PUT /*"] = upload_data;
+	};
+});
--- a/mod_list_inactive/mod_list_inactive.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_list_inactive/mod_list_inactive.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -19,6 +19,15 @@
 }
 
 function module.command(arg)
+	if #arg < 2 then
+		print("usage: prosodyctl mod_list_inactive example.net time [format]");
+		print("time is a number followed by 'day', 'week', 'month' or 'year'");
+		print("formats are:");
+		for name, fmt in pairs(output_formats) do
+			print(name, fmt:format("user@example.com", "last action"))
+		end
+		return;
+	end
 	local items = {};
 	local host = arg[1];
 	assert(hosts[host], "Host "..tostring(host).." does not exist");
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_log_rate/mod_log_rate.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -0,0 +1,17 @@
+module:set_global();
+
+local measure = require"core.statsmanager".measure;
+
+local function sink_maker(config)
+	local levels = {
+		debug = measure("rate", "log.debug");
+		info = measure("rate", "log.info");
+		warn = measure("rate", "log.warn");
+		error = measure("rate", "log.error");
+	};
+	return function (_, level)
+		return levels[level]();
+	end
+end
+
+require"core.loggingmanager".register_sink_type("measure", sink_maker);
--- a/mod_mam/mod_mam.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_mam/mod_mam.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -41,7 +41,7 @@
 	module:log("error", "Could not open archive storage");
 	return;
 elseif not archive.find then
-	module:log("error", "mod_%s does not support archiving, switch to mod_storage_sql2", archive._provided_by);
+	module:log("error", "mod_%s does not support archiving", archive._provided_by);
 	return;
 end
 
@@ -230,7 +230,9 @@
 	-- or that don't have a <body/>
 	or not stanza:get_child("body")
 	-- or if hints suggest we shouldn't
+	or stanza:get_child("no-permanent-storage", "urn:xmpp:hints") -- The XEP needs to decide on "store" or "storage"
 	or stanza:get_child("no-permanent-store", "urn:xmpp:hints")
+	or stanza:get_child("no-storage", "urn:xmpp:hints")
 	or stanza:get_child("no-store", "urn:xmpp:hints") then
 		module:log("debug", "Not archiving stanza: %s (content)", stanza:top_tag());
 		return;
@@ -247,6 +249,9 @@
 
 		-- And stash it
 		local ok, id = archive:append(store_user, nil, time_now(), with, stanza);
+		if ok then
+			module:fire_event("archive-message-added", { origin = origin, stanza = stanza, for_user = store_user, id = id });
+		end
 	else
 		module:log("debug", "Not archiving stanza: %s (prefs)", stanza:top_tag());
 	end
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_mamsub/mod_mamsub.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -0,0 +1,64 @@
+-- MAM Subscriptions prototype
+-- Copyright (C) 2015 Kim Alvefur
+--
+-- This file is MIT/X11 licensed.
+
+local mt = require"util.multitable";
+local st = require"util.stanza";
+
+local xmlns_mamsub = "http://prosody.im/protocol/mamsub";
+
+module:add_feature(xmlns_mamsub);
+
+local host_sessions = prosody.hosts[module.host].sessions;
+
+local weak = { __mode = "k" };
+
+module:hook("iq-set/self/"..xmlns_mamsub..":subscribe", function (event)
+	local origin, stanza = event.origin, event.stanza;
+	if origin.mamsub ~= nil then
+		origin.send(st.error_reply(stanza, "modify", "conflict"));
+		return true;
+	end
+	origin.mamsub = xmlns_mamsub;
+	local mamsub_sessions = host_sessions[origin.username].mamsub_sessions;
+	if not mamsub_sessions then
+		mamsub_sessions = setmetatable({}, weak);
+		host_sessions[origin.username].mamsub_sessions = mamsub_sessions;
+	end
+	mamsub_sessions[origin] = true;
+	origin.send(st.reply(stanza));
+	return true;
+end);
+
+module:hook("iq-set/self/"..xmlns_mamsub..":unsubscribe", function (event)
+	local origin, stanza = event.origin, event.stanza;
+	if origin.mamsub ~= xmlns_mamsub then
+		origin.send(st.error_reply(stanza, "modify", "conflict"));
+		return true;
+	end
+	origin.mamsub = nil;
+	local mamsub_sessions = host_sessions[origin.username].mamsub_sessions;
+	if mamsub_sessions then
+		mamsub_sessions[origin] = nil;
+	end
+	origin.send(st.reply(stanza));
+	return true;
+end);
+
+module:hook("archive-message-added", function (event)
+	local user_session = host_sessions[event.for_user];
+	local mamsub_sessions = user_session and user_session.mamsub_sessions;
+	if not mamsub_sessions then return end;
+
+	local for_broadcast = st.message():tag("mamsub", { xmlns = xmlns_mamsub })
+		:tag("forwarded", { xmlns = "urn:xmpp:forward:0" })
+			:add_child(event.stanza);
+
+	for session in pairs(mamsub_sessions) do
+		if session.mamsub == xmlns_mamsub then
+			for_broadcast.attr.to = session.full_jid;
+			session.send(for_broadcast);
+		end
+	end
+end);
--- a/mod_muc_limits/mod_muc_limits.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_muc_limits/mod_muc_limits.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -1,8 +1,8 @@
 
-local rooms = module:shared "muc/rooms";
+local mod_muc = module:depends"muc";
+local rooms = rawget(mod_muc, "rooms"); -- Old MUC API
 if not rooms then
-	module:log("error", "This module only works on MUC components!");
-	return;
+	rooms = module:shared"muc/rooms"; -- New MUC API
 end
 
 local jid_split, jid_bare = require "util.jid".split, require "util.jid".bare;
--- a/mod_profile/mod_profile.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_profile/mod_profile.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -1,7 +1,8 @@
 -- mod_profile
 
 local st = require"util.stanza";
-local jid_split, jid_bare = import("util.jid", "split", "bare");
+local jid_split = require"util.jid".split;
+local jid_bare = require"util.jid".bare;
 local is_admin = require"core.usermanager".is_admin;
 local vcard = require"util.vcard";
 local base64 = require"util.encodings".base64;
@@ -86,18 +87,25 @@
 	local username = origin.username;
 	local to = stanza.attr.to;
 	if to then username = jid_split(to); end
-	local data = storage:get(username);
+	local data, err = storage:get(username);
 	if not data then
+		if err then
+			origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err));
+			return true;
+		end
 		data = legacy_storage:get(username);
 		data = data and st.deserialize(data);
 		if data then
-			return origin.send(st.reply(stanza):add_child(data));
+			origin.send(st.reply(stanza):add_child(data));
+			return true;
 		end
 	end
 	if not data then
-		return origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
+		origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
+		return true;
 	end
-	return origin.send(st.reply(stanza):add_child(vcard.to_xep54(data)));
+	origin.send(st.reply(stanza):add_child(vcard.to_xep54(data)));
+	return true;
 end
 
 local function handle_set(event)
@@ -107,20 +115,23 @@
 	local to = stanza.attr.to;
 	if to then
 		if not is_admin(jid_bare(stanza.attr.from), module.host) then
-			return origin.send(st.error_reply(stanza, "auth", "forbidden"));
+			origin.send(st.error_reply(stanza, "auth", "forbidden"));
+			return true;
 		end
 		username = jid_split(to);
 	end
 	local ok, err = storage:set(username, data);
 	if not ok then
-		return origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err));
+		origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err));
+		return true;
 	end
 
 	if pep_plus and username then
 		update_pep(username, data);
 	end
 
-	return origin.send(st.reply(stanza));
+	origin.send(st.reply(stanza));
+	return true;
 end
 
 module:hook("iq-get/bare/vcard-temp:vCard", handle_get);
@@ -185,9 +196,11 @@
 		local username = jid_split(stanza.attr.to) or origin.username;
 		local data = storage:get(username);
 		if not data then
-			return origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
+			origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
+			return true;
 		end
-		return origin.send(st.reply(stanza):add_child(vcard.to_vcard4(data)));
+		origin.send(st.reply(stanza):add_child(vcard.to_vcard4(data)));
+		return true;
 	end);
 
 	if vcard.from_vcard4 then
@@ -195,14 +208,17 @@
 			local origin, stanza = event.origin, event.stanza;
 			local ok, err = storage:set(origin.username, vcard.from_vcard4(stanza.tags[1]));
 			if not ok then
-				return origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err));
+				origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err));
+				return true;
 			end
-			return origin.send(st.reply(stanza));
+			origin.send(st.reply(stanza));
+			return true;
 		end);
 	else
 		module:hook("iq-set/self/urn:ietf:params:xml:ns:vcard-4.0:vcard", function(event)
 			local origin, stanza = event.origin, event.stanza;
-			return origin.send(st.error_reply(stanza, "cancel", "feature-not-implemented"));
+			origin.send(st.error_reply(stanza, "cancel", "feature-not-implemented"));
+			return true;
 		end);
 	end
 end
--- a/mod_s2s_auth_dane/mod_s2s_auth_dane.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_s2s_auth_dane/mod_s2s_auth_dane.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -14,6 +14,8 @@
 -- No encryption offered
 -- Different hostname before and after STARTTLS - mod_s2s should complain
 -- Interaction with Dialback
+--
+-- luacheck: ignore module
 
 module:set_global();
 
@@ -294,7 +296,10 @@
 							log("info", "DANE validated ok for %s using %s", host, tlsa:getUsage());
 							if use == 2 then -- DANE-TA
 								session.cert_identity_status = "valid";
-								session.cert_chain_status = "valid";
+								if cert_verify_identity(host, "xmpp-server", cert) then
+									session.cert_chain_status = "valid";
+									-- else -- TODO Check against SRV target?
+								end
 								-- for usage 0, PKIX-CA, identity and chain has to be valid already
 							end
 							match_found = true;
--- a/mod_smacks/mod_smacks.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_smacks/mod_smacks.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -298,6 +298,18 @@
 	end
 end);
 
+local function handle_s2s_destroyed(event)
+	local session = event.session;
+	local queue = session.outgoing_stanza_queue;
+	if queue and #queue > 0 then
+		session.log("warn", "Destroying session with %d unacked stanzas", #queue);
+		handle_unacked_stanzas(session);
+	end
+end
+
+module:hook("s2sout-destroyed", handle_s2s_destroyed);
+module:hook("s2sin-destroyed", handle_s2s_destroyed);
+
 function handle_resume(session, stanza, xmlns_sm)
 	if session.full_jid then
 		session.log("warn", "Tried to resume after resource binding");
@@ -365,3 +377,18 @@
 end
 module:hook_stanza(xmlns_sm2, "resume", function (session, stanza) return handle_resume(session, stanza, xmlns_sm2); end);
 module:hook_stanza(xmlns_sm3, "resume", function (session, stanza) return handle_resume(session, stanza, xmlns_sm3); end);
+
+local function handle_read_timeout(event)
+	local session = event.session;
+	if session.smacks then
+		if session.awaiting_ack then
+			return false; -- Kick the session
+		end
+		(session.sends2s or session.send)(st.stanza("r", { xmlns = session.smacks }));
+		session.awaiting_ack = true;
+		return true;
+	end
+end
+
+module:hook("s2s-read-timeout", handle_read_timeout);
+module:hook("c2s-read-timeout", handle_read_timeout);
--- a/mod_smacks_offline/mod_smacks_offline.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_smacks_offline/mod_smacks_offline.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -21,9 +21,11 @@
 
 local host_sessions = prosody.hosts[module.host].sessions;
 mod_smacks.handle_unacked_stanzas = function (session)
-	local sessions = host_sessions[session.username].sessions;
-	if next(sessions) == session.resource and next(sessions, session.resource) == nil then
-		store_unacked_stanzas(session)
+	if session.username then
+		local sessions = host_sessions[session.username].sessions;
+		if next(sessions) == session.resource and next(sessions, session.resource) == nil then
+			store_unacked_stanzas(session)
+		end
 	end
 	return handle_unacked_stanzas(session);
 end
--- a/mod_storage_gdbm/mod_storage_gdbm.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_storage_gdbm/mod_storage_gdbm.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -73,7 +73,11 @@
 archive.get = keyval.get;
 archive.set = keyval.set;
 
-function archive:append(username, key, when, with, value)
+function archive:append(username, key, value, when, with)
+	if type(when) ~= "number" then
+		when, with, value = value, when, with;
+	end
+
 	key = key or uuid();
 	local meta = self:get(username);
 	if not meta then
@@ -148,7 +152,7 @@
 		db = assert(gdbm.open(db_path, "c"));
 		cache[db_path] = db;
 	end
-	return setmetatable({ _db = db; _path = db_path; store = store, typ = type }, driver_mt);
+	return setmetatable({ _db = db; _path = db_path; store = store, type = typ }, driver_mt);
 end
 
 function purge(_, user)
@@ -166,6 +170,7 @@
 function module.unload()
 	for db_path, db in pairs(cache) do
 		module:log("debug", "Closing db at %q", db_path);
+		gdbm.reorganize(db);
 		gdbm.sync(db);
 		gdbm.close(db);
 	end
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_storage_lmdb/mod_storage_lmdb.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -0,0 +1,86 @@
+-- mod_storage_lmdb
+-- Copyright (C) 2015 Kim Alvefur
+--
+-- This file is MIT/X11 licensed.
+-- 
+-- Depends on lightningdbm
+-- https://github.com/shmul/lightningdbm
+--
+-- luacheck: globals prosody open
+
+local lmdb = require"lightningmdb";
+local lfs = require"lfs";
+local path = require"util.paths";
+local serialization = require"util.serialization";
+local serialize = serialization.serialize;
+local deserialize = serialization.deserialize;
+
+local base_path = path.resolve_relative_path(prosody.paths.data, module.host);
+lfs.mkdir(base_path);
+
+local env = lmdb.env_create();
+assert(env:set_maxdbs(module:get_option_number("lmdb_maxdbs", 20)));
+local env_flags = 0;
+for i, flag in ipairs(module:get_option_array("lmdb_flags", {})) do
+	env_flags = env_flags + assert(lmdb["MDB_"..flag:upper()], "No such flag "..flag);
+end
+env:open(base_path, env_flags, tonumber("640", 8));
+
+local keyval = {};
+local keyval_mt = { __index = keyval, flags = lmdb.MDB_CREATE };
+
+function keyval:set(user, value)
+	local t = self.env:txn_begin(nil, 0);
+	if type(value) == "table" and next(value) == nil then
+		value = nil;
+	end
+	if value ~= nil then
+		value = serialize(value);
+	end
+	local ok, err;
+	if value ~= nil then
+		ok, err = t:put(self.db, user, value, 0);
+	else
+		ok, err = t:del(self.db, user, value);
+	end
+	if not ok then
+		t:abort();
+		return nil, err;
+	end
+	return t:commit();
+end
+
+function keyval:get(user)
+	local t = self.env:txn_begin(nil, 0);
+	local data, err = t:get(self.db, user, 0);
+	if not data then
+		t:abort();
+		return nil, err;
+	end
+	t:commit();
+	return deserialize(data);
+end
+
+local drivers = {
+	keyval = keyval_mt;
+}
+
+function open(_, store, typ)
+	typ = typ or "keyval";
+	local driver_mt = drivers[typ];
+	if not driver_mt then
+		return nil, "unsupported-store";
+	end
+	local t = env:txn_begin(nil, 0);
+	local db = t:dbi_open(store.."_"..typ, driver_mt.flags);
+	assert(t:commit());
+
+	return setmetatable({ env = env, store = store, type = typ, db = db }, driver_mt);
+end
+
+function module.unload()
+	env:sync(1);
+	env:close();
+end
+
+module:provides("storage");
--- a/mod_storage_memory/mod_storage_memory.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_storage_memory/mod_storage_memory.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -42,7 +42,10 @@
 local archive_store = {};
 archive_store.__index = archive_store;
 
-function archive_store:append(username, key, when, with, value)
+function archive_store:append(username, key, value, when, with)
+	if type(when) ~= "number" then
+		when, with, value = value, when, with;
+	end
 	local a = self.store[username];
 	if not a then
 		a = {};
@@ -116,7 +119,7 @@
 		i = old[i];
 		t = i.when;
 		if not(qstart >= t and qend <= t and (not qwith or i.with == qwith)) then
-			self:append(username, i.key, t, i.with, i.value);
+			self:append(username, i.key, i.value, t, i.with);
 		end
 	end
 	if #new == 0 then
--- a/mod_storage_muc_log/mod_storage_muc_log.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_storage_muc_log/mod_storage_muc_log.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -48,7 +48,10 @@
 	return with and tag.name .. "<" .. with or tag.name;
 end
 
-function driver:append(node, key, when, with, stanza)
+function driver:append(node, key, stanza, when, with)
+	if type(when) ~= "number" then
+		when, with, stanza = stanza, when, with;
+	end
 	local today = os_date(datef, when);
 	local now = os_date(timef, when);
 	local data = data_load(node, host, datastore .. "/" .. today) or {};
--- a/mod_storage_xmlarchive/mod_storage_xmlarchive.lua	Fri Jul 31 18:46:27 2015 +0200
+++ b/mod_storage_xmlarchive/mod_storage_xmlarchive.lua	Mon Aug 10 21:13:31 2015 +0200
@@ -1,3 +1,10 @@
+-- mod_storage_xmlarchive
+-- Copyright (C) 2015 Kim Alvefur
+--
+-- This file is MIT/X11 licensed.
+--
+-- luacheck: ignore unused self
+
 local dm = require "core.storagemanager".olddm;
 local hmac_sha256 = require"util.hashes".hmac_sha256;
 local st = require"util.stanza";
@@ -12,9 +19,9 @@
 	if not ok then
 		return ok, msg;
 	end
-	f:seek("set", offset);
-	return true;
+	return f:seek("set", offset);
 end;
+
 pcall(function()
 	local pposix = require "util.pposix";
 	fallocate = pposix.fallocate or fallocate;
@@ -23,32 +30,37 @@
 local archive = {};
 local archive_mt = { __index = archive };
 
-function archive:append(username, _, when, with, data)
+function archive:append(username, _, data, when, with)
+	if type(when) ~= "number" then
+		when, with, data = data, when, with;
+	end
 	if getmetatable(data) ~= st.stanza_mt then
+		module:log("error", "Attempt to store non-stanza object, traceback: %s", debug.traceback());
 		return nil, "unsupported-datatype";
 	end
+
 	username = username or "@";
 	data = tostring(data) .. "\n";
+
 	local day = dt.date(when);
 	local filename = dm.getpath(username.."@"..day, module.host, self.store, "xml", true);
+
 	local ok, err;
 	local f = io.open(filename, "r+");
 	if not f then
-		f, err = io.open(filename, "w");
-		if not f then return nil, err; end
+		f, err = io.open(filename, "w");     if not f then return nil, err; end
 		ok, err = dm.list_append(username, module.host, self.store, day);
 		if not ok then return nil, err; end
 	end
-	local offset = f and f:seek("end");
-	ok, err = fallocate(f, offset, #data);
-	if not ok then return nil, err; end
-	f:seek("set", offset);
-	ok, err = f:write(data);
-	if not ok then return nil, err; end
-	ok, err = f:close();
-	if not ok then return nil, err; end
+
+	local offset = f:seek("end"); -- Seek to the end and collect current file length
+	-- then make sure there is enough free space for what we're about to add
+	ok, err = fallocate(f, offset, #data); if not ok then return nil, err; end
+	ok, err = f:write(data);               if not ok then return nil, err; end
+	ok, err = f:close();                   if not ok then return nil, err; end
+
 	local id = day .. "-" .. hmac_sha256(username.."@"..day.."+"..offset, data, true):sub(-16);
-	ok, err = dm.list_append(username.."@"..day, module.host, self.store, { id = id, when = when, with = with, offset = offset, length = #data });
+	ok, err = dm.list_append(username.."@"..day, module.host, self.store, { id = id, when = dt.datetime(when), with = with, offset = offset, length = #data });
 	if not ok then return nil, err; end
 	return id;
 end
@@ -69,7 +81,13 @@
 	local stream = new_stream(stream_sess, { handlestanza = cb, stream_ns = "jabber:client"});
 	local dates = dm.list_load(username, module.host, self.store) or empty;
 	stream:feed(st.stanza("stream", { xmlns = "jabber:client" }):top_tag());
-	stream_sess.notopen = nil;
+	local function reset_stream()
+		stream:reset();
+		stream_sess.notopen = true;
+		stream:feed(st.stanza("stream", { xmlns = "jabber:client" }):top_tag());
+		stream_sess.notopen = nil;
+	end
+	reset_stream();
 
 	local limit = query.limit;
 	local start_day, step, last_day = 1, 1, #dates;
@@ -100,10 +118,11 @@
 
 	return function ()
 		if limit and count >= limit then xmlfile:close() return; end
+		local filename;
 
 		for d = start_day, last_day, step do
 			if d ~= start_day or not items then
-				module:log("debug", "Load items for %s", dates[d]);
+				module:log("debug", "Loading items from %s", dates[d]);
 				start_day = d;
 				items = dm.list_load(username .. "@" .. dates[d], module.host, self.store) or empty;
 				if not rev then
@@ -112,24 +131,33 @@
 					first_item, last_item = #items, 1;
 				end
 				local ferr;
-				xmlfile, ferr = io.open(dm.getpath(username .. "@" .. dates[d], module.host, self.store, "xml"));
+				filename = dm.getpath(username .. "@" .. dates[d], module.host, self.store, "xml");
+				xmlfile, ferr = io.open(filename);
 				if not xmlfile then
 					module:log("error", "Error: %s", ferr);
 					return;
 				end
 			end
 
+			local q_with, q_start, q_end = query.with, query.start, query["end"];
 			for i = first_item, last_item, step do
-				module:log("debug", "data[%q][%d]", dates[d], i);
 				local item = items[i];
+				local i_when, i_with = item.when, item.with;
+				if type(i_when) == "string" then
+					i_when = dt.parse(i_when);
+				end
+				if type(i_when) ~= "number" then
+					module:log("warn", "data[%q][%d].when is invalid", dates[d], i);
+					break;
+				end
 				if not item then
-					module:log("debug", "data[%q][%d] is nil", dates[d], i);
+					module:log("warn", "data[%q][%d] is nil", dates[d], i);
 					break;
 				end
 				if xmlfile and in_range
-				and (not query.with or item.with == query.with)
-				and (not query.start or item.when >= query.start)
-				and (not query["end"] or item.when <= query["end"]) then
+				and (not q_with or i_with == q_with)
+				and (not q_start or i_when >= q_start)
+				and (not q_end or i_when <= q_end) then
 					count = count + 1;
 					first_item = i + step;
 
@@ -137,12 +165,13 @@
 					local data = xmlfile:read(item.length);
 					local ok, err = stream:feed(data);
 					if not ok then
-						module:log("warn", "Parse error: %s", err);
+						module:log("warn", "Parse error in %s at %d+%d: %s", filename, item.offset, item.length, err);
+						reset_stream();
 					end
 					if result then
 						local stanza = result;
 						result = nil;
-						return item.id, stanza, item.when, item.with;
+						return item.id, stanza, i_when, i_with;
 					end
 				end
 				if (rev and item.id == query.after) or