prosody-modules: mod_storage_s3/mod_storage

annotate mod_storage_s3/mod_storage_s3.lua @ 5734:cef8bce2d71b

luacheck: Add new module API methods from trunk See * trunk rev 4d4f9e42bcf8 * trunk rev 65fb0d7a2312 * trunk rev c9ef35fab0b1

author	Kim Alvefur <zash@zash.se>
date	Mon, 27 Nov 2023 07:10:06 +0100
parents	b6518a71ca7e
children	9a1d8c39d0b0

rev	line source
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	1 local http = require "prosody.net.http";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	2 local array = require "prosody.util.array";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	3 local async = require "prosody.util.async";
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	4 local dt = require "prosody.util.datetime";
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	5 local hashes = require "prosody.util.hashes";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	6 local httputil = require "prosody.util.http";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	7 local it = require "prosody.util.iterators";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	8 local jid = require "prosody.util.jid";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	9 local json = require "prosody.util.json";
5697 4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	10 local promise = require "prosody.util.promise";
5733 b6518a71ca7e mod_storage_s3: Implement search for set of IDs Kim Alvefur <zash@zash.se> parents: 5732 diff changeset	11 local set = require "prosody.util.set";
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	12 local st = require "prosody.util.stanza";
5698 991fb904fa49 mod_storage_s3: Sort imports Kim Alvefur <zash@zash.se> parents: 5697 diff changeset	13 local uuid = require "prosody.util.uuid";
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	14 local xml = require "prosody.util.xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	15 local url = require "socket.url";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	16
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	17 local new_uuid = uuid.v7 or uuid.generate;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	18 local hmac_sha256 = hashes.hmac_sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	19 local sha256 = hashes.sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	20
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	21 local driver = {};
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	22
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	23 local bucket = module:get_option_string("s3_bucket", "prosody");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	24 local base_uri = module:get_option_string("s3_base_uri", "http://localhost:9000");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	25 local region = module:get_option_string("s3_region", "us-east-1");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	26
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	27 local access_key = module:get_option_string("s3_access_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	28 local secret_key = module:get_option_string("s3_secret_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	29
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	30 local aws4_format = "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	31
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	32 local function aws_auth(event)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	33 local request, options = event.request, event.options;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	34 local method = options.method or "GET";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	35 local query = options.query;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	36 local payload = options.body;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	37
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	38 local payload_type = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	39 if st.is_stanza(payload) then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	40 payload_type = "application/xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	41 payload = tostring(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	42 elseif payload ~= nil then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	43 payload_type = "application/json";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	44 payload = json.encode(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	45 end
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	46 options.body = payload;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	47
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	48 local payload_hash = sha256(payload or "", true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	49
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	50 local now = os.time();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	51 local aws_datetime = os.date("!%Y%m%dT%H%M%SZ", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	52 local aws_date = os.date("!%Y%m%d", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	53
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	54 local headers = {
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	55 ["Accept"] = "/";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	56 ["Authorization"] = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	57 ["Content-Type"] = payload_type;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	58 ["Host"] = request.authority;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	59 ["User-Agent"] = "Prosody XMPP Server";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	60 ["X-Amz-Content-Sha256"] = payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	61 ["X-Amz-Date"] = aws_datetime;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	62 };
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	63
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	64 local canonical_uri = url.build({ path = request.path });
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	65 local canonical_query = "";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	66 local canonical_headers = array();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	67 local signed_headers = array()
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	68
5670 2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	69 if query then
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	70 local sorted_query = array();
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	71 for name, value in it.sorted_pairs(query) do
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	72 sorted_query:push({ name = name; value = value });
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	73 end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	74 sorted_query:sort(function (a,b) return a.name < b.name end)
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	75 canonical_query = httputil.formencode(sorted_query):gsub("%%%x%x", string.upper);
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	76 request.query = canonical_query;
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	77 end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	78
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	79 for header_name, header_value in it.sorted_pairs(headers) do
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	80 header_name = header_name:lower();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	81 canonical_headers:push(header_name .. ":" .. header_value .. "\n");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	82 signed_headers:push(header_name);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	83 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	84
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	85 canonical_headers = canonical_headers:concat();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	86 signed_headers = signed_headers:concat(";");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	87
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	88 local scope = aws_date .. "/" .. region .. "/s3/aws4_request";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	89
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	90 local canonical_request = method .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	91 .. canonical_uri .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	92 .. canonical_query .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	93 .. canonical_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	94 .. signed_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	95 .. payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	96
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	97 local signature_payload = "AWS4-HMAC-SHA256" .. "\n" .. aws_datetime .. "\n" .. scope .. "\n" .. sha256(canonical_request, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	98
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	99 -- This can be cached?
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	100 local date_key = hmac_sha256("AWS4" .. secret_key, aws_date);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	101 local date_region_key = hmac_sha256(date_key, region);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	102 local date_region_service_key = hmac_sha256(date_region_key, "s3");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	103 local signing_key = hmac_sha256(date_region_service_key, "aws4_request");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	104
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	105 local signature = hmac_sha256(signing_key, signature_payload, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	106
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	107 headers["Authorization"] = string.format(aws4_format, access_key, scope, signed_headers, signature);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	108
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	109 options.headers = headers;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	110 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	111
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	112 function driver:open(store, typ)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	113 local mt = self[typ or "keyval"]
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	114 if not mt then
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	115 return nil, "unsupported-store";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	116 end
5700 9de7a1b36efb mod_storage_s3: Enable connection pooling added in latest trunk Kim Alvefur <zash@zash.se> parents: 5699 diff changeset	117 local httpclient = http.new({ connection_pooling = true });
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	118 httpclient.events.add_handler("pre-request", aws_auth);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	119 return setmetatable({ store = store; bucket = bucket; type = typ; http = httpclient }, mt);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	120 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	121
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	122 local keyval = { };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	123 driver.keyval = { __index = keyval; __name = module.name .. " keyval store" };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	124
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	125 local function new_request(self, method, path, query, payload)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	126 local request = url.parse(base_uri);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	127 request.path = path;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	128
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	129 return self.http:request(url.build(request), { method = method; body = payload; query = query });
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	130 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	131
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	132 -- coerce result back into Prosody data type
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	133 local function on_result(response)
5676 799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	134 if response.code == 404 and response.request.method == "GET" then
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	135 return nil;
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	136 end
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	137 if response.code >= 400 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	138 error(response.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	139 end
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	140 local content_type = response.headers["content-type"];
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	141 if content_type == "application/json" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	142 return json.decode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	143 elseif content_type == "application/xml" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	144 return xml.parse(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	145 elseif content_type == "application/x-www-form-urlencoded" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	146 return httputil.formdecode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	147 else
5699 83a2fb6df746 mod_storage_s3: Fix logging Kim Alvefur <zash@zash.se> parents: 5698 diff changeset	148 module:log("warn", "Unknown response data type %s", content_type);
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	149 return response.body;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	150 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	151 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	152
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	153 function keyval:_path(key)
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	154 return url.build_path({
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	155 is_absolute = true;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	156 bucket;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	157 jid.escape(module.host);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	158 jid.escape(self.store);
5675 17ea26cf7259 mod_storage_s3: Use '@' as placeholder for empty (host) store slots Kim Alvefur <zash@zash.se> parents: 5674 diff changeset	159 jid.escape(key or "@");
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	160 })
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	161 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	162
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	163 function keyval:get(user)
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	164 return async.wait_for(new_request(self, "GET", self:_path(user)):next(on_result));
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	165 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	166
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	167 function keyval:set(user, data)
5671 c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	168
c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	169 if data == nil or (type(data) == "table" and next(data) == nil) then
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	170 return async.wait_for(new_request(self, "DELETE", self:_path(user)));
5671 c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	171 end
c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	172
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	173 return async.wait_for(new_request(self, "PUT", self:_path(user), nil, data));
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	174 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	175
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	176 function keyval:users()
5672 c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	177 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	178 local prefix = url.build_path({ jid.escape(module.host); jid.escape(self.store); is_directory = true });
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	179 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, { prefix = prefix }))
5672 c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	180 if err or list_result.code ~= 200 then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	181 return nil, err;
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	182 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	183 local list_bucket_result = xml.parse(list_result.body);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	184 if list_bucket_result:get_child_text("IsTruncated") == "true" then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	185 local max_keys = list_bucket_result:get_child_text("MaxKeys");
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	186 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	187 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	188 local keys = array();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	189 for content in list_bucket_result:childtags("Contents") do
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	190 local key = url.parse_path(content:get_child_text("Key"));
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	191 keys:push(jid.unescape(key[3]));
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	192 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	193 return function()
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	194 return keys:pop();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	195 end
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	196 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	197
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	198 local archive = {};
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	199 driver.archive = { __index = archive };
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	200
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	201 archive.caps = {
5732 e8938a3166d2 mod_storage_s3: Advertise full id range archive query capability Kim Alvefur <zash@zash.se> parents: 5728 diff changeset	202 full_id_range = true; -- both before and after used
5733 b6518a71ca7e mod_storage_s3: Implement search for set of IDs Kim Alvefur <zash@zash.se> parents: 5732 diff changeset	203 ids = true;
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	204 };
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	205
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	206 function archive:_path(username, date, when, with, key)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	207 return url.build_path({
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	208 is_absolute = true;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	209 bucket;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	210 jid.escape(module.host);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	211 jid.escape(self.store);
5728 80702e33ba71 mod_storage_s3: Fix storing archives for host itself (e.g. mod_audit) Kim Alvefur <zash@zash.se> parents: 5700 diff changeset	212 jid.escape(username or "@");
80702e33ba71 mod_storage_s3: Fix storing archives for host itself (e.g. mod_audit) Kim Alvefur <zash@zash.se> parents: 5700 diff changeset	213 jid.escape(with and jid.prep(with) or "@");
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	214 date or dt.date(when);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	215 key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	216 })
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	217 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	218
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	219
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	220 -- PUT .../with/when/id
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	221 function archive:append(username, key, value, when, with)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	222 local wrapper = st.stanza("wrapper");
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	223 -- Minio had trouble with timestamps, probably the ':' characters, in paths.
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	224 wrapper:tag("delay", { xmlns = "urn:xmpp:delay"; stamp = dt.datetime(when) }):up();
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	225 wrapper:add_direct_child(value);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	226 key = key or new_uuid();
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	227 return async.wait_for(new_request(self, "PUT", self:_path(username, nil, when, with, key), nil, wrapper):next(function(r)
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	228 if r.code == 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	229 return key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	230 else
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	231 error(r.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	232 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	233 end));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	234 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	235
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	236 function archive:find(username, query)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	237 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	238 local prefix = { jid.escape(module.host); jid.escape(self.store); is_directory = true };
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	239 table.insert(prefix, jid.escape(username or "@"));
5674 51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	240 if not query then
51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	241 query = {};
51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	242 end
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	243 if query["with"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	244 table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	245 if query["start"] and query["end"] and dt.date(query["start"]) == dt.date(query["end"]) then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	246 table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	247 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	248 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	249
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	250 prefix = url.build_path(prefix);
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	251 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, {
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	252 prefix = prefix;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	253 ["max-keys"] = query["max"] and tostring(query["max"]);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	254 }));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	255 if err or list_result.code ~= 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	256 return nil, err;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	257 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	258 local list_bucket_result = xml.parse(list_result.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	259 if list_bucket_result:get_child_text("IsTruncated") == "true" then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	260 local max_keys = list_bucket_result:get_child_text("MaxKeys");
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	261 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	262 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	263 local keys = array();
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	264 local iterwrap = function(...)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	265 return ...;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	266 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	267 if query["reverse"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	268 query["before"], query["after"] = query["after"], query["before"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	269 iterwrap = it.reverse;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	270 end
5733 b6518a71ca7e mod_storage_s3: Implement search for set of IDs Kim Alvefur <zash@zash.se> parents: 5732 diff changeset	271 local ids = query["ids"] and set.new(query["ids"]);
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	272 local found = not query["after"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	273 for content in iterwrap(list_bucket_result:childtags("Contents")) do
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	274 local key = url.parse_path(content:get_child_text("Key"));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	275 if found and query["before"] == key[6] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	276 break
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	277 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	278 if (not query["with"] or query["with"] == jid.unescape(key[5]))
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	279 and (not query["start"] or dt.date(query["start"]) >= key[6])
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	280 and (not query["end"] or dt.date(query["end"]) <= key[6])
5733 b6518a71ca7e mod_storage_s3: Implement search for set of IDs Kim Alvefur <zash@zash.se> parents: 5732 diff changeset	281 and (not ids or ids:contains(key[6]))
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	282 and found then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	283 keys:push({ key = key[6]; date = key[5]; with = jid.unescape(key[4]) });
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	284 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	285 if not found and key[6] == query["after"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	286 found = not found
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	287 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	288 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	289 local i = 0;
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	290 local function get_next()
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	291 i = i + 1;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	292 local item = keys[i];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	293 if item == nil then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	294 return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	295 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	296 -- luacheck: ignore 431/err
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	297 local value, err = async.wait_for(new_request(self, "GET", self:_path(username or "@", item.date, nil, item.with, item.key)):next(on_result));
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	298 if not value then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	299 module:log("error", "%s", err);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	300 return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	301 end
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	302 local when = dt.parse(value:get_child_attr("delay", "urn:xmpp:delay", "stamp"));
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	303
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	304 if (not query["start"] or query["start"] >= when) and (not query["end"] or query["end"] <= when) then
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	305 return item.key, value.tags[2], when, item.with;
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	306 else
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	307 -- date was correct but not the time
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	308 return get_next();
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	309 end
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	310 end
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	311 return get_next;
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	312 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	313
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	314 function archive:users()
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	315 return it.unique(keyval.users(self));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	316 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	317
5697 4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	318 local function count(t) local n = 0; for _ in pairs(t) do n = n + 1; end return n; end
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	319
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	320 function archive:delete(username, query)
5697 4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	321 local deletions = {};
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	322 for key, _, when, with in self:find(username, query) do
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	323 deletions[key] = new_request(self, "DELETE", self:_path(username or "@", dt.date(when), nil, with, key));
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	324 end
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	325 return async.wait_for(promise.all(deletions):next(count));
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	326 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	327
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	328 module:provides("storage", driver);

Mercurial > prosody-modules

annotate mod_storage_s3/mod_storage_s3.lua @ 5734:cef8bce2d71b