Mercurial > prosody-modules

annotate mod_storage_lmdb/mod_storage_lmdb.lua @ 3900:06971a04216f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mod_muc_moderation: Add a copyright header
author Kim Alvefur <zash@zash.se>
date Sat, 22 Feb 2020 21:34:37 +0100
parents 2ed6f6eeeaba
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
1 -- mod_storage_lmdb
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
2 -- Copyright (C) 2015 Kim Alvefur
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
3 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
4 -- This file is MIT/X11 licensed.
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
5 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
6 -- Depends on lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
7 -- https://github.com/shmul/lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
8 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
9 -- luacheck: globals prosody open
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
10
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
11 local assert = assert;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
12 local select = select;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
13 local xpcall = xpcall;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
14 local traceback = debug.traceback;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
15
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
16 local lmdb = require"lightningmdb";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
17 local lfs = require"lfs";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
18 local path = require"util.paths";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
19 local serialization = require"util.serialization";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
20 local serialize = serialization.serialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
21 local deserialize = serialization.deserialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
22
1983
f7fc5088b340 mod_storage_lmdb: Allow passing a transaction flag so we can run get operations in a read-only transaction
Kim Alvefur <zash@zash.se>
parents: 1982
diff changeset
23 local function transaction(env, flag, func, ...)
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
24 local args, n_args = {...}, select("#", ...);
1983
f7fc5088b340 mod_storage_lmdb: Allow passing a transaction flag so we can run get operations in a read-only transaction
Kim Alvefur <zash@zash.se>
parents: 1982
diff changeset
25 local t = env:txn_begin(nil, flag);
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
26 local function f() return func(t, unpack(args, 1, n_args)); end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
27 local success, a, b, c = xpcall(f, traceback);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
28 if not success then
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
29 t:abort();
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
30 return success, a;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
31 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
32 local ok, err = t:commit();
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
33 if not ok then
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
34 return ok, err;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
35 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
36 return success, a, b, c;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
37 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
38
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
39 local function keyvalue_set(t, db, key, value)
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
40 if value ~= nil then
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
41 return assert(t:put(db, key, value, 0));
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
42 else
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
43 return t:del(db, key, value);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
44 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
45 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
46
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
47 local function keyvalue_get(t, db, key)
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
48 local data, err = t:get(db, key, 0);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
49 assert(data or not err, err);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
50 return data;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
51 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
52
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
53 local drivers = {};
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
54 local provider = {};
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
55
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
56 local keyval = {};
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
57 local keyval_mt = { __index = keyval, flags = lmdb.MDB_CREATE };
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
58 drivers.keyval = keyval_mt;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
59
1982
bb0b2eae5563 mod_storage_lmdb: Be consistent with argument name
Kim Alvefur <zash@zash.se>
parents: 1981
diff changeset
60 function keyval:set(key, value)
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
61 if type(value) == "table" and next(value) == nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
62 value = nil;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
63 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
64 if value ~= nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
65 value = serialize(value);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
66 end
1983
f7fc5088b340 mod_storage_lmdb: Allow passing a transaction flag so we can run get operations in a read-only transaction
Kim Alvefur <zash@zash.se>
parents: 1982
diff changeset
67 return transaction(self.env, 0, keyvalue_set, self.db, key, value);
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
68 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
69
1982
bb0b2eae5563 mod_storage_lmdb: Be consistent with argument name
Kim Alvefur <zash@zash.se>
parents: 1981
diff changeset
70 function keyval:get(key)
1983
f7fc5088b340 mod_storage_lmdb: Allow passing a transaction flag so we can run get operations in a read-only transaction
Kim Alvefur <zash@zash.se>
parents: 1982
diff changeset
71 local ok, data = transaction(self.env, lmdb.MDB_RDONLY, keyvalue_get, self.db, key);
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
72 if not ok then return ok, data; end
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
73 return deserialize(data);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
74 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
75
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
76 function provider:init(config)
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
77 if config.base_path then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
78 lfs.mkdir(config.base_path);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
79 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
80 local env = lmdb.env_create();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
81 env:set_maxdbs(config.maxdbs or 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
82 local env_flags = 0;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
83 if config.flags then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
84 for flag in config.flags do
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
85 env_flags = env_flags + assert(lmdb["MDB_"..flag:upper()], "No such flag "..flag);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
86 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
87 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
88 env:open(config.base_path or ".", env_flags, tonumber("640", 8));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
89 self.env = env;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
90 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
91
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
92 function provider:open(store, typ)
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
93 typ = typ or "keyval";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
94 local driver_mt = drivers[typ];
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
95 if not driver_mt then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
96 return nil, "unsupported-store";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
97 end
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
98 local env = self.env;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
99 local t = env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
100 local db = t:dbi_open(store.."_"..typ, driver_mt.flags);
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
101 local ok, err = t:commit();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
102 if not ok then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
103 module:log("error", "Could not open database %s_%s: %s", store, typ, tostring(err));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
104 return ok, err;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
105 end
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
106
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
107 return setmetatable({ env = env, store = store, type = typ, db = db }, driver_mt);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
108 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
109
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
110 if prosody then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
111 provider:init({
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
112 base_path = path.resolve_relative_path(prosody.paths.data, module.host);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
113 flags = module:get_option_set("lmdb_flags", {});
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
114 maxdbs = module:get_option_number("lmdb_maxdbs", 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
115 });
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
116
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
117 function module.unload() --luacheck: ignore
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
118 provider.env:sync(1);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
119 provider.env:close();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
120 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
121
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
122 module:provides("storage", provider);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
123 else
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
124 return provider;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
125 end