Mercurial > prosody-modules

annotate mod_anti_spam/rtbl.lib.lua @ 5906:cc30c4b5f006

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mod_audit_auth: Allow suppressing repeated failure/success log entries from the same IP for a time This can be triggered by e.g. a distributed brute force attack, or from Monal.
author Matthew Wild <mwild1@gmail.com>
date Mon, 13 May 2024 18:30:18 +0100
parents 259ffdbf8906
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
5859
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
1 local array = require "util.array";
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
2 local id = require "util.id";
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
3 local it = require "util.iterators";
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
4 local set = require "util.set";
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
5 local st = require "util.stanza";
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
6
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
7 module:depends("pubsub_subscription");
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
8
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
9 local function new_rtbl_subscription(rtbl_service_jid, rtbl_node, handlers)
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
10 local items = {};
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
11
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
12 local function notify(event_type, hash)
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
13 local handler = handlers[event_type];
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
14 if not handler then return; end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
15 handler(hash);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
16 end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
17
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
18 module:add_item("pubsub-subscription", {
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
19 service = rtbl_service_jid;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
20 node = rtbl_node;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
21
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
22 -- Callbacks:
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
23 on_subscribed = function()
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
24 module:log("info", "RTBL active: %s:%s", rtbl_service_jid, rtbl_node);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
25 end;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
26
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
27 on_error = function(err)
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
28 module:log(
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
29 "error",
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
30 "Failed to subscribe to RTBL: %s:%s %s::%s: %s",
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
31 rtbl_service_jid,
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
32 rtbl_node,
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
33 err.type,
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
34 err.condition,
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
35 err.text
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
36 );
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
37 end;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
38
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
39 on_item = function(event)
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
40 local hash = event.item.attr.id;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
41 if not hash then return; end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
42 module:log("debug", "Received new hash from %s:%s: %s", rtbl_service_jid, rtbl_node, hash);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
43 items[hash] = true;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
44 notify("added", hash);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
45 end;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
46
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
47 on_retract = function (event)
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
48 local hash = event.item.attr.id;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
49 if not hash then return; end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
50 module:log("debug", "Retracted hash from %s:%s: %s", rtbl_service_jid, rtbl_node, hash);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
51 items[hash] = nil;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
52 notify("removed", hash);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
53 end;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
54
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
55 purge = function()
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
56 module:log("debug", "Purge all hashes from %s:%s", rtbl_service_jid, rtbl_node);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
57 for hash in pairs(items) do
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
58 items[hash] = nil;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
59 notify("removed", hash);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
60 end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
61 end;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
62 });
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
63
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
64 local request_id = "rtbl-request-"..id.short();
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
65
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
66 local function request_list()
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
67 local items_request = st.iq({ to = rtbl_service_jid, from = module.host, type = "get", id = request_id })
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
68 :tag("pubsub", { xmlns = "http://jabber.org/protocol/pubsub" })
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
69 :tag("items", { node = rtbl_node }):up()
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
70 :up();
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
71 module:send(items_request);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
72 end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
73
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
74 local function update_list(event)
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
75 local from_jid = event.stanza.attr.from;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
76 if from_jid ~= rtbl_service_jid then
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
77 module:log("debug", "Ignoring RTBL response from unknown sender: %s", from_jid);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
78 return;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
79 end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
80 local items_el = event.stanza:find("{http://jabber.org/protocol/pubsub}pubsub/items");
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
81 if not items_el then
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
82 module:log("warn", "Invalid items response from RTBL service %s:%s", rtbl_service_jid, rtbl_node);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
83 return;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
84 end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
85
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
86 local old_entries = set.new(array.collect(it.keys(items)));
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
87
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
88 local n_added, n_removed, n_total = 0, 0, 0;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
89 for item in items_el:childtags("item") do
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
90 local hash = item.attr.id;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
91 if hash then
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
92 n_total = n_total + 1;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
93 if not old_entries:contains(hash) then
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
94 -- New entry
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
95 n_added = n_added + 1;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
96 items[hash] = true;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
97 notify("added", hash);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
98 else
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
99 -- Entry already existed
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
100 old_entries:remove(hash);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
101 end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
102 end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
103 end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
104
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
105 -- Remove old entries that weren't in the received list
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
106 for hash in old_entries do
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
107 n_removed = n_removed + 1;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
108 items[hash] = nil;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
109 notify("removed", hash);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
110 end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
111
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
112 module:log("info", "%d RTBL entries received from %s:%s (%d added, %d removed)", n_total, from_jid, rtbl_node, n_added, n_removed);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
113 return true;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
114 end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
115
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
116 module:hook("iq-result/host/"..request_id, update_list);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
117 module:add_timer(0, request_list);
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
118 end
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
119
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
120 return {
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
121 new_rtbl_subscription = new_rtbl_subscription;
259ffdbf8906 mod_anti_spam: New module for spam filtering (pre-alpha)
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
122 }