prosody-modules: mod_storage_s3/mod_storage

annotate mod_storage_s3/mod_storage_s3.lua @ 5696:66986f5271c3

mod_storage_s3: Skip archive items matching on date but not full datetime Since it only encodes dates in paths, it would have returned items from outside the specified start..end range if they were from earlier or later in the same (UTC) day.

author	Kim Alvefur <zash@zash.se>
date	Sat, 11 Nov 2023 22:26:39 +0100
parents	b4632d5f840b
children	4a0279c5c7ed

rev	line source
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	1 local http = require "prosody.net.http";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	2 local array = require "prosody.util.array";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	3 local async = require "prosody.util.async";
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	4 local dt = require "prosody.util.datetime";
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	5 local hashes = require "prosody.util.hashes";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	6 local httputil = require "prosody.util.http";
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	7 local uuid = require "prosody.util.uuid";
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	8 local it = require "prosody.util.iterators";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	9 local jid = require "prosody.util.jid";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	10 local json = require "prosody.util.json";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	11 local st = require "prosody.util.stanza";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	12 local xml = require "prosody.util.xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	13 local url = require "socket.url";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	14
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	15 local new_uuid = uuid.v7 or uuid.generate;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	16 local hmac_sha256 = hashes.hmac_sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	17 local sha256 = hashes.sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	18
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	19 local driver = {};
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	20
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	21 local bucket = module:get_option_string("s3_bucket", "prosody");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	22 local base_uri = module:get_option_string("s3_base_uri", "http://localhost:9000");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	23 local region = module:get_option_string("s3_region", "us-east-1");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	24
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	25 local access_key = module:get_option_string("s3_access_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	26 local secret_key = module:get_option_string("s3_secret_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	27
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	28 local aws4_format = "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	29
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	30 local function aws_auth(event)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	31 local request, options = event.request, event.options;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	32 local method = options.method or "GET";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	33 local query = options.query;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	34 local payload = options.body;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	35
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	36 local payload_type = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	37 if st.is_stanza(payload) then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	38 payload_type = "application/xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	39 payload = tostring(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	40 elseif payload ~= nil then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	41 payload_type = "application/json";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	42 payload = json.encode(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	43 end
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	44 options.body = payload;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	45
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	46 local payload_hash = sha256(payload or "", true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	47
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	48 local now = os.time();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	49 local aws_datetime = os.date("!%Y%m%dT%H%M%SZ", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	50 local aws_date = os.date("!%Y%m%d", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	51
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	52 local headers = {
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	53 ["Accept"] = "/";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	54 ["Authorization"] = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	55 ["Content-Type"] = payload_type;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	56 ["Host"] = request.authority;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	57 ["User-Agent"] = "Prosody XMPP Server";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	58 ["X-Amz-Content-Sha256"] = payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	59 ["X-Amz-Date"] = aws_datetime;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	60 };
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	61
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	62 local canonical_uri = url.build({ path = request.path });
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	63 local canonical_query = "";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	64 local canonical_headers = array();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	65 local signed_headers = array()
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	66
5670 2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	67 if query then
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	68 local sorted_query = array();
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	69 for name, value in it.sorted_pairs(query) do
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	70 sorted_query:push({ name = name; value = value });
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	71 end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	72 sorted_query:sort(function (a,b) return a.name < b.name end)
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	73 canonical_query = httputil.formencode(sorted_query):gsub("%%%x%x", string.upper);
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	74 request.query = canonical_query;
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	75 end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	76
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	77 for header_name, header_value in it.sorted_pairs(headers) do
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	78 header_name = header_name:lower();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	79 canonical_headers:push(header_name .. ":" .. header_value .. "\n");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	80 signed_headers:push(header_name);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	81 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	82
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	83 canonical_headers = canonical_headers:concat();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	84 signed_headers = signed_headers:concat(";");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	85
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	86 local scope = aws_date .. "/" .. region .. "/s3/aws4_request";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	87
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	88 local canonical_request = method .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	89 .. canonical_uri .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	90 .. canonical_query .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	91 .. canonical_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	92 .. signed_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	93 .. payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	94
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	95 local signature_payload = "AWS4-HMAC-SHA256" .. "\n" .. aws_datetime .. "\n" .. scope .. "\n" .. sha256(canonical_request, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	96
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	97 -- This can be cached?
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	98 local date_key = hmac_sha256("AWS4" .. secret_key, aws_date);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	99 local date_region_key = hmac_sha256(date_key, region);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	100 local date_region_service_key = hmac_sha256(date_region_key, "s3");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	101 local signing_key = hmac_sha256(date_region_service_key, "aws4_request");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	102
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	103 local signature = hmac_sha256(signing_key, signature_payload, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	104
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	105 headers["Authorization"] = string.format(aws4_format, access_key, scope, signed_headers, signature);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	106
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	107 options.headers = headers;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	108 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	109
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	110 function driver:open(store, typ)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	111 local mt = self[typ or "keyval"]
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	112 if not mt then
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	113 return nil, "unsupported-store";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	114 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	115 local httpclient = http.new({});
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	116 httpclient.events.add_handler("pre-request", aws_auth);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	117 return setmetatable({ store = store; bucket = bucket; type = typ; http = httpclient }, mt);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	118 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	119
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	120 local keyval = { };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	121 driver.keyval = { __index = keyval; __name = module.name .. " keyval store" };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	122
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	123 local function new_request(self, method, path, query, payload)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	124 local request = url.parse(base_uri);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	125 request.path = path;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	126
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	127 return self.http:request(url.build(request), { method = method; body = payload; query = query });
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	128 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	129
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	130 -- coerce result back into Prosody data type
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	131 local function on_result(response)
5676 799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	132 if response.code == 404 and response.request.method == "GET" then
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	133 return nil;
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	134 end
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	135 if response.code >= 400 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	136 error(response.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	137 end
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	138 local content_type = response.headers["content-type"];
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	139 if content_type == "application/json" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	140 return json.decode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	141 elseif content_type == "application/xml" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	142 return xml.parse(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	143 elseif content_type == "application/x-www-form-urlencoded" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	144 return httputil.formdecode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	145 else
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	146 response.log("warn", "Unknown response data type %s", content_type);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	147 return response.body;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	148 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	149 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	150
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	151 function keyval:_path(key)
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	152 return url.build_path({
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	153 is_absolute = true;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	154 bucket;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	155 jid.escape(module.host);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	156 jid.escape(self.store);
5675 17ea26cf7259 mod_storage_s3: Use '@' as placeholder for empty (host) store slots Kim Alvefur <zash@zash.se> parents: 5674 diff changeset	157 jid.escape(key or "@");
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	158 })
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	159 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	160
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	161 function keyval:get(user)
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	162 return async.wait_for(new_request(self, "GET", self:_path(user)):next(on_result));
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	163 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	164
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	165 function keyval:set(user, data)
5671 c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	166
c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	167 if data == nil or (type(data) == "table" and next(data) == nil) then
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	168 return async.wait_for(new_request(self, "DELETE", self:_path(user)));
5671 c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	169 end
c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	170
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	171 return async.wait_for(new_request(self, "PUT", self:_path(user), nil, data));
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	172 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	173
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	174 function keyval:users()
5672 c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	175 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	176 local prefix = url.build_path({ jid.escape(module.host); jid.escape(self.store); is_directory = true });
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	177 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, { prefix = prefix }))
5672 c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	178 if err or list_result.code ~= 200 then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	179 return nil, err;
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	180 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	181 local list_bucket_result = xml.parse(list_result.body);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	182 if list_bucket_result:get_child_text("IsTruncated") == "true" then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	183 local max_keys = list_bucket_result:get_child_text("MaxKeys");
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	184 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	185 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	186 local keys = array();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	187 for content in list_bucket_result:childtags("Contents") do
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	188 local key = url.parse_path(content:get_child_text("Key"));
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	189 keys:push(jid.unescape(key[3]));
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	190 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	191 return function()
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	192 return keys:pop();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	193 end
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	194 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	195
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	196 local archive = {};
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	197 driver.archive = { __index = archive };
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	198
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	199 archive.caps = {
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	200 };
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	201
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	202 function archive:_path(username, date, when, with, key)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	203 return url.build_path({
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	204 is_absolute = true;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	205 bucket;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	206 jid.escape(module.host);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	207 jid.escape(self.store);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	208 jid.escape(username);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	209 jid.escape(jid.prep(with));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	210 date or dt.date(when);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	211 key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	212 })
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	213 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	214
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	215
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	216 -- PUT .../with/when/id
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	217 function archive:append(username, key, value, when, with)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	218 local wrapper = st.stanza("wrapper");
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	219 -- Minio had trouble with timestamps, probably the ':' characters, in paths.
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	220 wrapper:tag("delay", { xmlns = "urn:xmpp:delay"; stamp = dt.datetime(when) }):up();
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	221 wrapper:add_direct_child(value);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	222 key = key or new_uuid();
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	223 return async.wait_for(new_request(self, "PUT", self:_path(username, nil, when, with, key), nil, wrapper):next(function(r)
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	224 if r.code == 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	225 return key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	226 else
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	227 error(r.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	228 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	229 end));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	230 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	231
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	232 function archive:find(username, query)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	233 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	234 local prefix = { jid.escape(module.host); jid.escape(self.store); is_directory = true };
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	235 table.insert(prefix, jid.escape(username or "@"));
5674 51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	236 if not query then
51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	237 query = {};
51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	238 end
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	239 if query["with"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	240 table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	241 if query["start"] and query["end"] and dt.date(query["start"]) == dt.date(query["end"]) then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	242 table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	243 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	244 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	245
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	246 prefix = url.build_path(prefix);
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	247 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, {
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	248 prefix = prefix;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	249 ["max-keys"] = query["max"] and tostring(query["max"]);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	250 }));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	251 if err or list_result.code ~= 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	252 return nil, err;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	253 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	254 local list_bucket_result = xml.parse(list_result.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	255 if list_bucket_result:get_child_text("IsTruncated") == "true" then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	256 local max_keys = list_bucket_result:get_child_text("MaxKeys");
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	257 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	258 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	259 local keys = array();
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	260 local iterwrap = function(...)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	261 return ...;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	262 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	263 if query["reverse"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	264 query["before"], query["after"] = query["after"], query["before"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	265 iterwrap = it.reverse;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	266 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	267 local found = not query["after"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	268 for content in iterwrap(list_bucket_result:childtags("Contents")) do
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	269 local key = url.parse_path(content:get_child_text("Key"));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	270 if found and query["before"] == key[6] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	271 break
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	272 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	273 if (not query["with"] or query["with"] == jid.unescape(key[5]))
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	274 and (not query["start"] or dt.date(query["start"]) >= key[6])
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	275 and (not query["end"] or dt.date(query["end"]) <= key[6])
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	276 and found then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	277 keys:push({ key = key[6]; date = key[5]; with = jid.unescape(key[4]) });
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	278 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	279 if not found and key[6] == query["after"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	280 found = not found
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	281 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	282 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	283 local i = 0;
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	284 local function get_next()
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	285 i = i + 1;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	286 local item = keys[i];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	287 if item == nil then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	288 return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	289 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	290 -- luacheck: ignore 431/err
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	291 local value, err = async.wait_for(new_request(self, "GET", self:_path(username or "@", item.date, nil, item.with, item.key)):next(on_result));
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	292 if not value then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	293 module:log("error", "%s", err);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	294 return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	295 end
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	296 local when = dt.parse(value:get_child_attr("delay", "urn:xmpp:delay", "stamp"));
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	297
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	298 if (not query["start"] or query["start"] >= when) and (not query["end"] or query["end"] <= when) then
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	299 return item.key, value.tags[2], when, item.with;
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	300 else
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	301 -- date was correct but not the time
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	302 return get_next();
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	303 end
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	304 end
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	305 return get_next;
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	306 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	307
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	308 function archive:users()
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	309 return it.unique(keyval.users(self));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	310 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	311
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	312 --[[ TODO
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	313 function archive:delete(username, query)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	314 return nil, "not-implemented";
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	315 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	316 --]]
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	317
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	318 module:provides("storage", driver);

Mercurial > prosody-modules

annotate mod_storage_s3/mod_storage_s3.lua @ 5696:66986f5271c3