Mercurial > prosody-modules

annotate mod_storage_multi/mod_storage_multi.lua @ 4922:816b23e09c20

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mod_rest: Improve logging of results from callback startup probe Especially interested in logging 500 status codes and their errors, to diagnose problems with the callback API.
author Kim Alvefur <zash@zash.se>
date Sat, 09 Apr 2022 00:43:18 +0200
parents 8c4a2d85e8bf
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1492
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
1 -- mod_storage_multi
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
2
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
3 local storagemanager = require"core.storagemanager";
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
4 local backends = module:get_option_array(module.name); -- TODO better name?
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
5
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
6 -- TODO migrate data "upwards"
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
7
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
8 -- one → one successful write is success
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
9 -- all → all backends must report success
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
10 -- majority → majority of backends must report success
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
11 local policy = module:get_option_string(module.name.."_policy", "all");
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
12
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
13 local keyval_store = {};
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
14 keyval_store.__index = keyval_store;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
15
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
16 function keyval_store:get(username)
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
17 local backends = self.backends;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
18 local data, err;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
19 for i = 1, #backends do
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
20 module:log("debug", "%s:%s:get(%q)", tostring(backends[i].get), backends[i]._store, username);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
21 data, err = backends[i]:get(username);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
22 if err then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
23 module:log("error", tostring(err));
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
24 elseif not data then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
25 module:log("debug", "No data returned");
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
26 else
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
27 module:log("debug", "Data returned");
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
28 return data, err;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
29 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
30 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
31 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
32
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
33 -- This is where it gets complicated
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
34 function keyval_store:set(username, data)
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
35 local backends = self.backends;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
36 local ok, err, backend;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
37 local all, one, oks = true, false, 0;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
38 for i = 1, #backends do
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
39 backend = backends[i];
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
40 module:log("debug", "%s:%s:set(%q)", tostring(backends[i].get), backends[i].store, username);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
41 ok, err = backend:set(username, data);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
42 if not ok then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
43 module:log("error", "Error in storage driver %s: %s", backend.name, tostring(err));
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
44 else
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
45 oks = oks + 1;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
46 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
47 one = one or ok; -- At least one successful write
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
48 all = all and ok; -- All successful
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
49 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
50 if policy == "all" then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
51 return all, err
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
52 elseif policy == "majority" then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
53 return oks > (#backends/2), err;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
54 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
55 -- elseif policy == "one" then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
56 return one, err;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
57 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
58
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
59 local stores = {
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
60 keyval = keyval_store;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
61 }
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
62
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
63 local driver = {};
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
64
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
65 function driver:open(store, typ)
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
66 local store_mt = stores[typ or "keyval"];
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
67 if store_mt then
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
68 local my_backends = {};
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
69 local driver, opened
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
70 for i = 1, #backends do
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
71 driver = storagemanager.load_driver(module.host, backends[i]);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
72 opened = driver:open(store, typ);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
73 my_backends[i] = assert(driver:open(store, typ));
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
74 my_backends[i]._store = store;
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
75 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
76 return setmetatable({ backends = my_backends }, store_mt);
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
77 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
78 return nil, "unsupported-store";
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
79 end
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
80
8c4a2d85e8bf mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff changeset
81 module:provides("storage", driver);