Mercurial > prosody-modules
annotate mod_mam_archive/mod_mam_archive.lua @ 4876:0f5f2d4475b9
mod_http_xep227: Add support for import via APIs rather than direct store manipulation
In particular this transitions PEP nodes and data to be imported via mod_pep's
APIs, fixing issues with importing at runtime while PEP data may already be
live in RAM.
Next obvious candidate for this approach is rosters, so clients get immediate
roster pushes and other special handling (such as emitting subscribes to reach
the desired subscription state).
| author | Matthew Wild <mwild1@gmail.com> |
|---|---|
| date | Tue, 18 Jan 2022 17:01:18 +0000 |
| parents | 3e97dae28215 |
| children | 8ff308fad9fd |
| rev | line source |
|---|---|
| 1471 | 1 -- Prosody IM |
| 2 -- | |
| 3 -- This project is MIT/X11 licensed. Please see the | |
| 4 -- COPYING file in the source package for more information. | |
| 5 -- | |
| 6 local get_prefs = module:require"mod_mam/mamprefs".get; | |
| 7 local set_prefs = module:require"mod_mam/mamprefs".set; | |
|
2706
3e97dae28215
mod_mam_archive: Use util.rsm (fixes #877, depends on recent 0.10+)
Dennis Schridde <devurandom@gmx.net>
parents:
1586
diff
changeset
|
8 local rsm = require "util.rsm"; |
| 1471 | 9 local jid_bare = require "util.jid".bare; |
| 10 local jid_prep = require "util.jid".prep; | |
| 11 local date_parse = require "util.datetime".parse; | |
|
1498
e82592ed744b
mod_mam_archive: Applying @vstakhov 's patch (https://gist.github.com/vstakhov/306ea813a38021dcf3d4).
syn@syn.im
parents:
1477
diff
changeset
|
12 local date_format = require "util.datetime".datetime; |
| 1471 | 13 |
| 14 local st = require "util.stanza"; | |
| 15 local archive_store = "archive2"; | |
| 16 local archive = module:open_store(archive_store, "archive"); | |
| 17 local global_default_policy = module:get_option("default_archive_policy", false); | |
| 18 local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50); | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
19 local conversation_interval = tonumber(module:get_option_number("archive_conversation_interval", 86400)); |
|
1498
e82592ed744b
mod_mam_archive: Applying @vstakhov 's patch (https://gist.github.com/vstakhov/306ea813a38021dcf3d4).
syn@syn.im
parents:
1477
diff
changeset
|
20 local resolve_relative_path = require "core.configmanager".resolve_relative_path; |
| 1471 | 21 |
| 22 -- Feature discovery | |
| 23 local xmlns_archive = "urn:xmpp:archive" | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
24 local feature_archive = st.stanza("feature", {xmlns=xmlns_archive}):tag("optional"); |
| 1471 | 25 if(global_default_policy) then |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
26 feature_archive:tag("default"); |
| 1471 | 27 end |
| 28 module:add_extension(feature_archive); | |
| 29 module:add_feature("urn:xmpp:archive:auto"); | |
| 30 module:add_feature("urn:xmpp:archive:manage"); | |
| 31 module:add_feature("urn:xmpp:archive:pref"); | |
| 32 module:add_feature("http://jabber.org/protocol/rsm"); | |
| 33 -- -------------------------------------------------- | |
| 34 | |
| 35 local function prefs_to_stanza(prefs) | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
36 local prefstanza = st.stanza("pref", { xmlns="urn:xmpp:archive" }); |
| 1471 | 37 local default = prefs[false] ~= nil and prefs[false] or global_default_policy; |
| 38 | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
39 prefstanza:tag("default", {otr="oppose", save=default and "true" or "false"}):up(); |
|
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
40 prefstanza:tag("method", {type="auto", use="concede"}):up(); |
|
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
41 prefstanza:tag("method", {type="local", use="concede"}):up(); |
|
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
42 prefstanza:tag("method", {type="manual", use="concede"}):up(); |
| 1471 | 43 |
| 44 for jid, choice in pairs(prefs) do | |
| 45 if jid then | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
46 prefstanza:tag("item", {jid=jid, otr="prefer", save=choice and "message" or "false" }):up() |
| 1471 | 47 end |
| 48 end | |
| 49 | |
| 50 return prefstanza; | |
| 51 end | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
52 local function prefs_from_stanza(stanza, username) |
|
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
53 local current_prefs = get_prefs(username); |
| 1471 | 54 |
| 55 -- "default" | "item" | "session" | "method" | |
| 56 for elem in stanza:children() do | |
| 57 if elem.name == "default" then | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
58 current_prefs[false] = elem.attr["save"] == "true"; |
| 1471 | 59 elseif elem.name == "item" then |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
60 current_prefs[elem.attr["jid"]] = not elem.attr["save"] == "false"; |
| 1471 | 61 elseif elem.name == "session" then |
| 62 module:log("info", "element is not supported: " .. tostring(elem)); | |
| 63 -- local found = false; | |
| 64 -- for child in data:children() do | |
| 65 -- if child.name == elem.name and child.attr["thread"] == elem.attr["thread"] then | |
| 66 -- for k, v in pairs(elem.attr) do | |
| 67 -- child.attr[k] = v; | |
| 68 -- end | |
| 69 -- found = true; | |
| 70 -- break; | |
| 71 -- end | |
| 72 -- end | |
| 73 -- if not found then | |
| 74 -- data:tag(elem.name, elem.attr):up(); | |
| 75 -- end | |
| 76 elseif elem.name == "method" then | |
| 77 module:log("info", "element is not supported: " .. tostring(elem)); | |
| 78 -- local newpref = stanza.tags[1]; -- iq:pref | |
| 79 -- for _, e in ipairs(newpref.tags) do | |
| 80 -- -- if e.name ~= "method" then continue end | |
| 81 -- local found = false; | |
| 82 -- for child in data:children() do | |
| 83 -- if child.name == "method" and child.attr["type"] == e.attr["type"] then | |
| 84 -- child.attr["use"] = e.attr["use"]; | |
| 85 -- found = true; | |
| 86 -- break; | |
| 87 -- end | |
| 88 -- end | |
| 89 -- if not found then | |
| 90 -- data:tag(e.name, e.attr):up(); | |
| 91 -- end | |
| 92 -- end | |
| 93 end | |
| 94 end | |
| 95 end | |
| 96 | |
| 97 ------------------------------------------------------------ | |
| 98 -- Preferences | |
| 99 ------------------------------------------------------------ | |
| 100 local function preferences_handler(event) | |
| 101 local origin, stanza = event.origin, event.stanza; | |
| 102 local user = origin.username; | |
| 103 local reply = st.reply(stanza); | |
| 104 | |
| 105 if stanza.attr.type == "get" then | |
| 106 reply:add_child(prefs_to_stanza(get_prefs(user))); | |
| 107 end | |
| 108 if stanza.attr.type == "set" then | |
| 109 local new_prefs = stanza:get_child("pref", xmlns_archive); | |
| 110 if not new_prefs then return false; end | |
| 111 | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
112 local prefs = prefs_from_stanza(stanza, origin.username); |
| 1471 | 113 local ok, err = set_prefs(user, prefs); |
| 114 | |
| 115 if not ok then | |
| 116 return origin.send(st.error_reply(stanza, "cancel", "internal-server-error", "Error storing preferences: "..tostring(err))); | |
| 117 end | |
| 118 end | |
| 119 return origin.send(reply); | |
| 120 end | |
| 121 local function auto_handler(event) | |
| 122 local origin, stanza = event.origin, event.stanza; | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
123 if not stanza.attr["type"] == "set" then return false; end |
| 1471 | 124 |
| 125 local user = origin.username; | |
| 126 local prefs = get_prefs(user); | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
127 local auto = stanza:get_child("auto", xmlns_archive); |
| 1471 | 128 |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
129 prefs[false] = auto.attr["save"] ~= nil and auto.attr["save"] == "true" or false; |
| 1471 | 130 set_prefs(user, prefs); |
| 131 | |
| 132 return origin.send(st.reply(stanza)); | |
| 133 end | |
| 134 | |
| 135 -- excerpt from mod_storage_sql2 | |
| 136 local function get_db() | |
| 137 local mod_sql = module:require("sql"); | |
| 138 local params = module:get_option("sql"); | |
| 139 local engine; | |
| 140 | |
| 141 params = params or { driver = "SQLite3" }; | |
| 142 if params.driver == "SQLite3" then | |
| 143 params.database = resolve_relative_path(prosody.paths.data or ".", params.database or "prosody.sqlite"); | |
| 144 end | |
| 145 | |
| 146 assert(params.driver and params.database, "Both the SQL driver and the database need to be specified"); | |
| 147 engine = mod_sql:create_engine(params); | |
| 148 engine:set_encoding(); | |
| 149 | |
| 150 return engine; | |
| 151 end | |
| 152 | |
| 153 ------------------------------------------------------------ | |
| 154 -- Collections. In our case there is one conversation with each contact for the whole day for simplicity | |
| 155 ------------------------------------------------------------ | |
| 156 local function list_stanza_to_query(origin, list_el) | |
| 157 local sql = "SELECT `with`, `when` / ".. conversation_interval .." as `day`, COUNT(0) FROM `prosodyarchive` WHERE `host`=? AND `user`=? AND `store`=? "; | |
| 158 local args = {origin.host, origin.username, archive_store}; | |
| 159 | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
160 local with = list_el.attr["with"]; |
| 1471 | 161 if with ~= nil then |
| 162 sql = sql .. "AND `with` = ? "; | |
| 163 table.insert(args, jid_bare(with)); | |
| 164 end | |
| 165 | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
166 local after = list_el.attr["start"]; |
| 1471 | 167 if after ~= nil then |
| 1586 | 168 sql = sql .. "AND `when` >= ? "; |
| 1471 | 169 table.insert(args, date_parse(after)); |
| 170 end | |
| 171 | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
172 local before = list_el.attr["end"]; |
| 1471 | 173 if before ~= nil then |
| 174 sql = sql .. "AND `when` <= ? "; | |
| 175 table.insert(args, date_parse(before)); | |
| 176 end | |
| 177 | |
| 178 sql = sql .. "GROUP BY `with`, `when` / ".. conversation_interval .." ORDER BY `when` / ".. conversation_interval .." ASC "; | |
| 179 | |
| 180 local qset = rsm.get(list_el); | |
| 181 local limit = math.min(qset and qset.max or default_max_items, max_max_items); | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
182 sql = sql.."LIMIT ?"; |
| 1471 | 183 table.insert(args, limit); |
| 184 | |
| 185 table.insert(args, 1, sql); | |
| 186 return args; | |
| 187 end | |
| 188 local function list_handler(event) | |
| 189 local db = get_db(); | |
| 190 local origin, stanza = event.origin, event.stanza; | |
| 191 local reply = st.reply(stanza); | |
| 192 | |
| 193 local query = list_stanza_to_query(origin, stanza.tags[1]); | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
194 local list = reply:tag("list", {xmlns=xmlns_archive}); |
| 1471 | 195 |
| 196 for row in db:select(unpack(query)) do | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
197 list:tag("chat", { |
| 1471 | 198 xmlns=xmlns_archive, |
| 199 with=row[1], | |
| 200 start=date_format(row[2] * conversation_interval), | |
| 201 version=row[3] | |
| 202 }):up(); | |
| 203 end | |
| 204 | |
| 205 origin.send(reply); | |
| 206 return true; | |
| 207 end | |
| 208 | |
| 209 ------------------------------------------------------------ | |
| 210 -- Message archive retrieval | |
| 211 ------------------------------------------------------------ | |
| 212 | |
| 213 local function retrieve_handler(event) | |
| 214 local origin, stanza = event.origin, event.stanza; | |
| 215 local reply = st.reply(stanza); | |
| 216 | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
217 local retrieve = stanza:get_child("retrieve", xmlns_archive); |
| 1471 | 218 |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
219 local qwith = retrieve.attr["with"]; |
|
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
220 local qstart = retrieve.attr["start"]; |
| 1471 | 221 |
| 222 module:log("debug", "Archive query, with %s from %s)", | |
| 223 qwith or "anyone", qstart or "the dawn of time"); | |
| 224 | |
| 225 if qstart then -- Validate timestamps | |
| 226 local vstart = (qstart and date_parse(qstart)); | |
| 227 if (qstart and not vstart) then | |
| 228 origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid timestamp")) | |
| 229 return true | |
| 230 end | |
| 231 qstart = vstart; | |
| 232 end | |
| 233 | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
234 if qwith then -- Validate the "with" jid |
| 1471 | 235 local pwith = qwith and jid_prep(qwith); |
| 236 if pwith and not qwith then -- it failed prepping | |
| 237 origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid JID")) | |
| 238 return true | |
| 239 end | |
| 240 qwith = jid_bare(pwith); | |
| 241 end | |
| 242 | |
| 243 -- RSM stuff | |
| 244 local qset = rsm.get(retrieve); | |
| 245 local qmax = math.min(qset and qset.max or default_max_items, max_max_items); | |
| 246 local reverse = qset and qset.before or false; | |
| 247 local before, after = qset and qset.before, qset and qset.after; | |
| 248 if type(before) ~= "string" then before = nil; end | |
| 249 | |
| 250 -- Load all the data! | |
| 251 local data, err = archive:find(origin.username, { | |
| 252 start = qstart; ["end"] = qstart + conversation_interval; | |
| 253 with = qwith; | |
| 254 limit = qmax; | |
| 255 before = before; after = after; | |
| 256 reverse = reverse; | |
| 1523 | 257 total = true; |
| 1471 | 258 }); |
| 259 | |
| 260 if not data then | |
| 261 return origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err)); | |
| 262 end | |
| 263 local count = err; | |
| 264 | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
265 local chat = reply:tag("chat", {xmlns=xmlns_archive, with=qwith, start=date_format(qstart), version=count}); |
|
1498
e82592ed744b
mod_mam_archive: Applying @vstakhov 's patch (https://gist.github.com/vstakhov/306ea813a38021dcf3d4).
syn@syn.im
parents:
1477
diff
changeset
|
266 local first, last; |
| 1523 | 267 |
| 268 module:log("debug", "Count "..count); | |
| 1471 | 269 for id, item, when in data do |
|
1477
db870913e1cf
mod_mam_archive: Doing stanza deserialization after mod_storage the right way
syn@syn.im
parents:
1476
diff
changeset
|
270 if not getmetatable(item) == st.stanza_mt then |
|
db870913e1cf
mod_mam_archive: Doing stanza deserialization after mod_storage the right way
syn@syn.im
parents:
1476
diff
changeset
|
271 item = st.deserialize(item); |
|
db870913e1cf
mod_mam_archive: Doing stanza deserialization after mod_storage the right way
syn@syn.im
parents:
1476
diff
changeset
|
272 end |
|
db870913e1cf
mod_mam_archive: Doing stanza deserialization after mod_storage the right way
syn@syn.im
parents:
1476
diff
changeset
|
273 module:log("debug", tostring(item)); |
|
db870913e1cf
mod_mam_archive: Doing stanza deserialization after mod_storage the right way
syn@syn.im
parents:
1476
diff
changeset
|
274 |
|
1498
e82592ed744b
mod_mam_archive: Applying @vstakhov 's patch (https://gist.github.com/vstakhov/306ea813a38021dcf3d4).
syn@syn.im
parents:
1477
diff
changeset
|
275 local tag = jid_bare(item.attr["from"]) == jid_bare(origin.full_jid) and "to" or "from"; |
| 1471 | 276 tag = chat:tag(tag, {secs = when - qstart}); |
|
1477
db870913e1cf
mod_mam_archive: Doing stanza deserialization after mod_storage the right way
syn@syn.im
parents:
1476
diff
changeset
|
277 tag:add_child(item:get_child("body")):up(); |
|
1498
e82592ed744b
mod_mam_archive: Applying @vstakhov 's patch (https://gist.github.com/vstakhov/306ea813a38021dcf3d4).
syn@syn.im
parents:
1477
diff
changeset
|
278 if not first then first = id; end |
|
e82592ed744b
mod_mam_archive: Applying @vstakhov 's patch (https://gist.github.com/vstakhov/306ea813a38021dcf3d4).
syn@syn.im
parents:
1477
diff
changeset
|
279 last = id; |
| 1471 | 280 end |
|
1498
e82592ed744b
mod_mam_archive: Applying @vstakhov 's patch (https://gist.github.com/vstakhov/306ea813a38021dcf3d4).
syn@syn.im
parents:
1477
diff
changeset
|
281 reply:add_child(rsm.generate{ first = first, last = last, count = count }) |
| 1471 | 282 |
| 283 origin.send(reply); | |
| 284 return true; | |
| 285 end | |
| 286 | |
| 287 local function not_implemented(event) | |
| 288 local origin, stanza = event.origin, event.stanza; | |
|
1476
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
289 local reply = st.reply(stanza):tag("error", {type="cancel"}); |
|
08ca6dd36e39
mod_mam_archive: Fixing issues noted in code review for 153df603f73d3b69c434f2790cff0270de14bb75
syn@syn.im
parents:
1471
diff
changeset
|
290 reply:tag("feature-not-implemented", {xmlns="urn:ietf:params:xml:ns:xmpp-stanzas"}):up(); |
| 1471 | 291 origin.send(reply); |
| 292 end | |
| 293 | |
| 294 -- Preferences | |
| 295 module:hook("iq/self/urn:xmpp:archive:pref", preferences_handler); | |
| 296 module:hook("iq/self/urn:xmpp:archive:auto", auto_handler); | |
| 297 module:hook("iq/self/urn:xmpp:archive:itemremove", not_implemented); | |
| 298 module:hook("iq/self/urn:xmpp:archive:sessionremove", not_implemented); | |
| 299 | |
| 300 -- Message Archive Management | |
| 301 module:hook("iq/self/urn:xmpp:archive:list", list_handler); | |
| 302 module:hook("iq/self/urn:xmpp:archive:retrieve", retrieve_handler); | |
| 303 module:hook("iq/self/urn:xmpp:archive:remove", not_implemented); | |
| 304 | |
| 305 -- manual archiving | |
| 306 module:hook("iq/self/urn:xmpp:archive:save", not_implemented); | |
| 307 -- replication | |
| 308 module:hook("iq/self/urn:xmpp:archive:modified", not_implemented); |