Mercurial > prosody-modules

annotate mod_storage_multi/mod_storage_multi.lua @ 5536:96dec7681af8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mod_firewall: Update user marks to store instantly via map store The original approach was to keep marks in memory only, and persist them at shutdown. That saves I/O, at the cost of potentially losing marks on an unclean shutdown. This change persists marks instantly, which may have some performance overhead but should be more "correct". It also splits the marking/unmarking into an event which may be watched or even fired by other modules.
author Matthew Wild <mwild1@gmail.com>
date Thu, 08 Jun 2023 16:20:42 +0100
parents 8c4a2d85e8bf
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1492
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
1 -- mod_storage_multi
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
2
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
3 local storagemanager = require"core.storagemanager";
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
4 local backends = module:get_option_array(module.name); -- TODO better name?
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
5
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
6 -- TODO migrate data "upwards"
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
7
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
8 -- one → one successful write is success
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
9 -- all → all backends must report success
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
10 -- majority → majority of backends must report success
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
11 local policy = module:get_option_string(module.name.."_policy", "all");
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
12
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
13 local keyval_store = {};
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
14 keyval_store.__index = keyval_store;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
15
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
16 function keyval_store:get(username)
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
17 local backends = self.backends;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
18 local data, err;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
19 for i = 1, #backends do
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
20 module:log("debug", "%s:%s:get(%q)", tostring(backends[i].get), backends[i]._store, username);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
21 data, err = backends[i]:get(username);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
22 if err then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
23 module:log("error", tostring(err));
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
24 elseif not data then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
25 module:log("debug", "No data returned");
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
26 else
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
27 module:log("debug", "Data returned");
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
28 return data, err;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
29 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
30 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
31 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
32
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
33 -- This is where it gets complicated
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
34 function keyval_store:set(username, data)
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
35 local backends = self.backends;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
36 local ok, err, backend;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
37 local all, one, oks = true, false, 0;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
38 for i = 1, #backends do
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
39 backend = backends[i];
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
40 module:log("debug", "%s:%s:set(%q)", tostring(backends[i].get), backends[i].store, username);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
41 ok, err = backend:set(username, data);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
42 if not ok then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
43 module:log("error", "Error in storage driver %s: %s", backend.name, tostring(err));
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
44 else
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
45 oks = oks + 1;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
46 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
47 one = one or ok; -- At least one successful write
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
48 all = all and ok; -- All successful
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
49 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
50 if policy == "all" then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
51 return all, err
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
52 elseif policy == "majority" then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
53 return oks > (#backends/2), err;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
54 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
55 -- elseif policy == "one" then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
56 return one, err;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
57 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
58
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
59 local stores = {
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
60 keyval = keyval_store;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
61 }
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
62
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
63 local driver = {};
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
64
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
65 function driver:open(store, typ)
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
66 local store_mt = stores[typ or "keyval"];
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
67 if store_mt then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
68 local my_backends = {};
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
69 local driver, opened
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
70 for i = 1, #backends do
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
71 driver = storagemanager.load_driver(module.host, backends[i]);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
72 opened = driver:open(store, typ);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
73 my_backends[i] = assert(driver:open(store, typ));
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
74 my_backends[i]._store = store;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
75 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
76 return setmetatable({ backends = my_backends }, store_mt);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
77 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
78 return nil, "unsupported-store";
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
79 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
80
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
81 module:provides("storage", driver);