Mercurial > prosody-modules

annotate mod_storage_lmdb/mod_storage_lmdb.lua @ 5139:449e4ca4de32

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mod_unified_push: Remove dependency on trunk util.jwt (0.12 compat) This should allow the module to work on 0.12, while preserving expiry checking (which was not built in to 0.12's util.jwt).
author Matthew Wild <mwild1@gmail.com>
date Tue, 10 Jan 2023 16:34:21 +0000
parents 2ed6f6eeeaba
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
1 -- mod_storage_lmdb
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
2 -- Copyright (C) 2015 Kim Alvefur
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
3 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
4 -- This file is MIT/X11 licensed.
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
5 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
6 -- Depends on lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
7 -- https://github.com/shmul/lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
8 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
9 -- luacheck: globals prosody open
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
10
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
11 local assert = assert;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
12 local select = select;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
13 local xpcall = xpcall;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
14 local traceback = debug.traceback;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
15
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
16 local lmdb = require"lightningmdb";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
17 local lfs = require"lfs";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
18 local path = require"util.paths";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
19 local serialization = require"util.serialization";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
20 local serialize = serialization.serialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
21 local deserialize = serialization.deserialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
22
1983
f7fc5088b340 mod_storage_lmdb: Allow passing a transaction flag so we can run get operations in a read-only transaction
Kim Alvefur <zash@zash.se>
parents: 1982
diff changeset
23 local function transaction(env, flag, func, ...)
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
24 local args, n_args = {...}, select("#", ...);
1983
f7fc5088b340 mod_storage_lmdb: Allow passing a transaction flag so we can run get operations in a read-only transaction
Kim Alvefur <zash@zash.se>
parents: 1982
diff changeset
25 local t = env:txn_begin(nil, flag);
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
26 local function f() return func(t, unpack(args, 1, n_args)); end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
27 local success, a, b, c = xpcall(f, traceback);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
28 if not success then
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
29 t:abort();
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
30 return success, a;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
31 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
32 local ok, err = t:commit();
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
33 if not ok then
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
34 return ok, err;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
35 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
36 return success, a, b, c;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
37 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
38
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
39 local function keyvalue_set(t, db, key, value)
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
40 if value ~= nil then
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
41 return assert(t:put(db, key, value, 0));
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
42 else
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
43 return t:del(db, key, value);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
44 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
45 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
46
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
47 local function keyvalue_get(t, db, key)
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
48 local data, err = t:get(db, key, 0);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
49 assert(data or not err, err);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
50 return data;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
51 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
52
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
53 local drivers = {};
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
54 local provider = {};
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
55
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
56 local keyval = {};
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
57 local keyval_mt = { __index = keyval, flags = lmdb.MDB_CREATE };
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
58 drivers.keyval = keyval_mt;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
59
1982
bb0b2eae5563 mod_storage_lmdb: Be consistent with argument name
Kim Alvefur <zash@zash.se>
parents: 1981
diff changeset
60 function keyval:set(key, value)
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
61 if type(value) == "table" and next(value) == nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
62 value = nil;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
63 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
64 if value ~= nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
65 value = serialize(value);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
66 end
1983
f7fc5088b340 mod_storage_lmdb: Allow passing a transaction flag so we can run get operations in a read-only transaction
Kim Alvefur <zash@zash.se>
parents: 1982
diff changeset
67 return transaction(self.env, 0, keyvalue_set, self.db, key, value);
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
68 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
69
1982
bb0b2eae5563 mod_storage_lmdb: Be consistent with argument name
Kim Alvefur <zash@zash.se>
parents: 1981
diff changeset
70 function keyval:get(key)
1983
f7fc5088b340 mod_storage_lmdb: Allow passing a transaction flag so we can run get operations in a read-only transaction
Kim Alvefur <zash@zash.se>
parents: 1982
diff changeset
71 local ok, data = transaction(self.env, lmdb.MDB_RDONLY, keyvalue_get, self.db, key);
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
72 if not ok then return ok, data; end
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
73 return deserialize(data);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
74 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
75
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
76 function provider:init(config)
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
77 if config.base_path then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
78 lfs.mkdir(config.base_path);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
79 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
80 local env = lmdb.env_create();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
81 env:set_maxdbs(config.maxdbs or 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
82 local env_flags = 0;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
83 if config.flags then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
84 for flag in config.flags do
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
85 env_flags = env_flags + assert(lmdb["MDB_"..flag:upper()], "No such flag "..flag);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
86 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
87 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
88 env:open(config.base_path or ".", env_flags, tonumber("640", 8));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
89 self.env = env;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
90 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
91
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
92 function provider:open(store, typ)
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
93 typ = typ or "keyval";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
94 local driver_mt = drivers[typ];
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
95 if not driver_mt then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
96 return nil, "unsupported-store";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
97 end
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
98 local env = self.env;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
99 local t = env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
100 local db = t:dbi_open(store.."_"..typ, driver_mt.flags);
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
101 local ok, err = t:commit();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
102 if not ok then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
103 module:log("error", "Could not open database %s_%s: %s", store, typ, tostring(err));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
104 return ok, err;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
105 end
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
106
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
107 return setmetatable({ env = env, store = store, type = typ, db = db }, driver_mt);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
108 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
109
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
110 if prosody then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
111 provider:init({
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
112 base_path = path.resolve_relative_path(prosody.paths.data, module.host);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
113 flags = module:get_option_set("lmdb_flags", {});
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
114 maxdbs = module:get_option_number("lmdb_maxdbs", 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
115 });
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
116
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
117 function module.unload() --luacheck: ignore
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
118 provider.env:sync(1);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
119 provider.env:close();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
120 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
121
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
122 module:provides("storage", provider);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
123 else
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
124 return provider;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
125 end