Mercurial > prosody-modules

annotate mod_storage_lmdb/mod_storage_lmdb.lua @ 1881:3683eb95bc1a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mod_smacks: Add experimental resending of unacked messages in s2s queues (disabled by default)
author Kim Alvefur <zash@zash.se>
date Tue, 29 Sep 2015 21:54:40 +0200
parents 669d1208221a
children 1f815f57fa57
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
1 -- mod_storage_lmdb
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
2 -- Copyright (C) 2015 Kim Alvefur
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
3 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
4 -- This file is MIT/X11 licensed.
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
5 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
6 -- Depends on lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
7 -- https://github.com/shmul/lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
8 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
9 -- luacheck: globals prosody open
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
10
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
11 local lmdb = require"lightningmdb";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
12 local lfs = require"lfs";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
13 local path = require"util.paths";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
14 local serialization = require"util.serialization";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
15 local serialize = serialization.serialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
16 local deserialize = serialization.deserialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
17
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
18 local drivers = {};
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
19 local provider = {};
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
20
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
21 local keyval = {};
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
22 local keyval_mt = { __index = keyval, flags = lmdb.MDB_CREATE };
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
23 drivers.keyval = keyval_mt;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
24
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
25 function keyval:set(user, value)
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
26 local t = self.env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
27 if type(value) == "table" and next(value) == nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
28 value = nil;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
29 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
30 if value ~= nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
31 value = serialize(value);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
32 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
33 local ok, err;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
34 if value ~= nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
35 ok, err = t:put(self.db, user, value, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
36 else
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
37 ok, err = t:del(self.db, user, value);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
38 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
39 if not ok then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
40 t:abort();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
41 return nil, err;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
42 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
43 return t:commit();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
44 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
45
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
46 function keyval:get(user)
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
47 local t = self.env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
48 local data, err = t:get(self.db, user, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
49 if not data then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
50 t:abort();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
51 return nil, err;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
52 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
53 t:commit();
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
54 return deserialize(data);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
55 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
56
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
57 function provider:init(config)
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
58 if config.base_path then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
59 lfs.mkdir(config.base_path);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
60 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
61 local env = lmdb.env_create();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
62 env:set_maxdbs(config.maxdbs or 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
63 local env_flags = 0;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
64 if config.flags then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
65 for flag in config.flags do
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
66 env_flags = env_flags + assert(lmdb["MDB_"..flag:upper()], "No such flag "..flag);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
67 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
68 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
69 env:open(config.base_path or ".", env_flags, tonumber("640", 8));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
70 self.env = env;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
71 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
72
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
73 function provider:open(store, typ)
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
74 typ = typ or "keyval";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
75 local driver_mt = drivers[typ];
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
76 if not driver_mt then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
77 return nil, "unsupported-store";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
78 end
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
79 local env = self.env;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
80 local t = env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
81 local db = t:dbi_open(store.."_"..typ, driver_mt.flags);
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
82 local ok, err = t:commit();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
83 if not ok then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
84 module:log("error", "Could not open database %s_%s: %s", store, typ, tostring(err));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
85 return ok, err;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
86 end
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
87
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
88 return setmetatable({ env = env, store = store, type = typ, db = db }, driver_mt);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
89 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
90
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
91 if prosody then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
92 provider:init({
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
93 base_path = path.resolve_relative_path(prosody.paths.data, module.host);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
94 flags = module:get_option_set("lmdb_flags", {});
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
95 maxdbs = module:get_option_number("lmdb_maxdbs", 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
96 });
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
97
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
98 function module.unload()
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
99 provider.env:sync(1);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
100 provider.env:close();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
101 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
102
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
103 module:provides("storage", provider);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
104 else
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
105 return provider;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
106 end