prosody-modules: mod_storage_s3/mod_storage

annotate mod_storage_s3/mod_storage_s3.lua @ 5715:8488ebde5739

mod_http_oauth2: Skip consent screen if requested by client and same scopes already granted This follows the intent behind the OpenID Connect 'prompt' parameter when it does not include the 'consent' keyword, that is the client wishes to skip the consent screen. If the user has already granted the exact same scopes to the exact same client in the past, then one can assume that they may grant it again.

author	Kim Alvefur <zash@zash.se>
date	Tue, 14 Nov 2023 23:03:37 +0100
parents	9de7a1b36efb
children	80702e33ba71

rev	line source
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	1 local http = require "prosody.net.http";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	2 local array = require "prosody.util.array";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	3 local async = require "prosody.util.async";
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	4 local dt = require "prosody.util.datetime";
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	5 local hashes = require "prosody.util.hashes";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	6 local httputil = require "prosody.util.http";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	7 local it = require "prosody.util.iterators";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	8 local jid = require "prosody.util.jid";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	9 local json = require "prosody.util.json";
5697 4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	10 local promise = require "prosody.util.promise";
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	11 local st = require "prosody.util.stanza";
5698 991fb904fa49 mod_storage_s3: Sort imports Kim Alvefur <zash@zash.se> parents: 5697 diff changeset	12 local uuid = require "prosody.util.uuid";
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	13 local xml = require "prosody.util.xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	14 local url = require "socket.url";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	15
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	16 local new_uuid = uuid.v7 or uuid.generate;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	17 local hmac_sha256 = hashes.hmac_sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	18 local sha256 = hashes.sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	19
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	20 local driver = {};
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	21
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	22 local bucket = module:get_option_string("s3_bucket", "prosody");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	23 local base_uri = module:get_option_string("s3_base_uri", "http://localhost:9000");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	24 local region = module:get_option_string("s3_region", "us-east-1");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	25
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	26 local access_key = module:get_option_string("s3_access_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	27 local secret_key = module:get_option_string("s3_secret_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	28
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	29 local aws4_format = "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	30
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	31 local function aws_auth(event)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	32 local request, options = event.request, event.options;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	33 local method = options.method or "GET";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	34 local query = options.query;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	35 local payload = options.body;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	36
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	37 local payload_type = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	38 if st.is_stanza(payload) then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	39 payload_type = "application/xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	40 payload = tostring(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	41 elseif payload ~= nil then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	42 payload_type = "application/json";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	43 payload = json.encode(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	44 end
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	45 options.body = payload;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	46
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	47 local payload_hash = sha256(payload or "", true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	48
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	49 local now = os.time();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	50 local aws_datetime = os.date("!%Y%m%dT%H%M%SZ", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	51 local aws_date = os.date("!%Y%m%d", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	52
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	53 local headers = {
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	54 ["Accept"] = "/";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	55 ["Authorization"] = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	56 ["Content-Type"] = payload_type;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	57 ["Host"] = request.authority;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	58 ["User-Agent"] = "Prosody XMPP Server";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	59 ["X-Amz-Content-Sha256"] = payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	60 ["X-Amz-Date"] = aws_datetime;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	61 };
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	62
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	63 local canonical_uri = url.build({ path = request.path });
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	64 local canonical_query = "";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	65 local canonical_headers = array();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	66 local signed_headers = array()
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	67
5670 2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	68 if query then
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	69 local sorted_query = array();
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	70 for name, value in it.sorted_pairs(query) do
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	71 sorted_query:push({ name = name; value = value });
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	72 end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	73 sorted_query:sort(function (a,b) return a.name < b.name end)
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	74 canonical_query = httputil.formencode(sorted_query):gsub("%%%x%x", string.upper);
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	75 request.query = canonical_query;
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	76 end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	77
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	78 for header_name, header_value in it.sorted_pairs(headers) do
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	79 header_name = header_name:lower();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	80 canonical_headers:push(header_name .. ":" .. header_value .. "\n");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	81 signed_headers:push(header_name);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	82 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	83
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	84 canonical_headers = canonical_headers:concat();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	85 signed_headers = signed_headers:concat(";");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	86
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	87 local scope = aws_date .. "/" .. region .. "/s3/aws4_request";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	88
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	89 local canonical_request = method .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	90 .. canonical_uri .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	91 .. canonical_query .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	92 .. canonical_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	93 .. signed_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	94 .. payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	95
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	96 local signature_payload = "AWS4-HMAC-SHA256" .. "\n" .. aws_datetime .. "\n" .. scope .. "\n" .. sha256(canonical_request, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	97
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	98 -- This can be cached?
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	99 local date_key = hmac_sha256("AWS4" .. secret_key, aws_date);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	100 local date_region_key = hmac_sha256(date_key, region);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	101 local date_region_service_key = hmac_sha256(date_region_key, "s3");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	102 local signing_key = hmac_sha256(date_region_service_key, "aws4_request");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	103
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	104 local signature = hmac_sha256(signing_key, signature_payload, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	105
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	106 headers["Authorization"] = string.format(aws4_format, access_key, scope, signed_headers, signature);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	107
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	108 options.headers = headers;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	109 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	110
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	111 function driver:open(store, typ)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	112 local mt = self[typ or "keyval"]
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	113 if not mt then
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	114 return nil, "unsupported-store";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	115 end
5700 9de7a1b36efb mod_storage_s3: Enable connection pooling added in latest trunk Kim Alvefur <zash@zash.se> parents: 5699 diff changeset	116 local httpclient = http.new({ connection_pooling = true });
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	117 httpclient.events.add_handler("pre-request", aws_auth);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	118 return setmetatable({ store = store; bucket = bucket; type = typ; http = httpclient }, mt);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	119 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	120
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	121 local keyval = { };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	122 driver.keyval = { __index = keyval; __name = module.name .. " keyval store" };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	123
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	124 local function new_request(self, method, path, query, payload)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	125 local request = url.parse(base_uri);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	126 request.path = path;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	127
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	128 return self.http:request(url.build(request), { method = method; body = payload; query = query });
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	129 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	130
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	131 -- coerce result back into Prosody data type
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	132 local function on_result(response)
5676 799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	133 if response.code == 404 and response.request.method == "GET" then
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	134 return nil;
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	135 end
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	136 if response.code >= 400 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	137 error(response.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	138 end
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	139 local content_type = response.headers["content-type"];
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	140 if content_type == "application/json" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	141 return json.decode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	142 elseif content_type == "application/xml" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	143 return xml.parse(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	144 elseif content_type == "application/x-www-form-urlencoded" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	145 return httputil.formdecode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	146 else
5699 83a2fb6df746 mod_storage_s3: Fix logging Kim Alvefur <zash@zash.se> parents: 5698 diff changeset	147 module:log("warn", "Unknown response data type %s", content_type);
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	148 return response.body;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	149 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	150 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	151
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	152 function keyval:_path(key)
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	153 return url.build_path({
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	154 is_absolute = true;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	155 bucket;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	156 jid.escape(module.host);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	157 jid.escape(self.store);
5675 17ea26cf7259 mod_storage_s3: Use '@' as placeholder for empty (host) store slots Kim Alvefur <zash@zash.se> parents: 5674 diff changeset	158 jid.escape(key or "@");
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	159 })
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	160 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	161
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	162 function keyval:get(user)
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	163 return async.wait_for(new_request(self, "GET", self:_path(user)):next(on_result));
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	164 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	165
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	166 function keyval:set(user, data)
5671 c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	167
c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	168 if data == nil or (type(data) == "table" and next(data) == nil) then
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	169 return async.wait_for(new_request(self, "DELETE", self:_path(user)));
5671 c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	170 end
c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	171
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	172 return async.wait_for(new_request(self, "PUT", self:_path(user), nil, data));
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	173 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	174
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	175 function keyval:users()
5672 c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	176 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	177 local prefix = url.build_path({ jid.escape(module.host); jid.escape(self.store); is_directory = true });
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	178 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, { prefix = prefix }))
5672 c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	179 if err or list_result.code ~= 200 then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	180 return nil, err;
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	181 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	182 local list_bucket_result = xml.parse(list_result.body);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	183 if list_bucket_result:get_child_text("IsTruncated") == "true" then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	184 local max_keys = list_bucket_result:get_child_text("MaxKeys");
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	185 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	186 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	187 local keys = array();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	188 for content in list_bucket_result:childtags("Contents") do
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	189 local key = url.parse_path(content:get_child_text("Key"));
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	190 keys:push(jid.unescape(key[3]));
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	191 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	192 return function()
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	193 return keys:pop();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	194 end
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	195 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	196
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	197 local archive = {};
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	198 driver.archive = { __index = archive };
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	199
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	200 archive.caps = {
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	201 };
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	202
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	203 function archive:_path(username, date, when, with, key)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	204 return url.build_path({
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	205 is_absolute = true;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	206 bucket;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	207 jid.escape(module.host);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	208 jid.escape(self.store);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	209 jid.escape(username);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	210 jid.escape(jid.prep(with));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	211 date or dt.date(when);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	212 key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	213 })
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	214 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	215
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	216
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	217 -- PUT .../with/when/id
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	218 function archive:append(username, key, value, when, with)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	219 local wrapper = st.stanza("wrapper");
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	220 -- Minio had trouble with timestamps, probably the ':' characters, in paths.
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	221 wrapper:tag("delay", { xmlns = "urn:xmpp:delay"; stamp = dt.datetime(when) }):up();
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	222 wrapper:add_direct_child(value);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	223 key = key or new_uuid();
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	224 return async.wait_for(new_request(self, "PUT", self:_path(username, nil, when, with, key), nil, wrapper):next(function(r)
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	225 if r.code == 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	226 return key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	227 else
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	228 error(r.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	229 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	230 end));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	231 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	232
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	233 function archive:find(username, query)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	234 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	235 local prefix = { jid.escape(module.host); jid.escape(self.store); is_directory = true };
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	236 table.insert(prefix, jid.escape(username or "@"));
5674 51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	237 if not query then
51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	238 query = {};
51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	239 end
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	240 if query["with"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	241 table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	242 if query["start"] and query["end"] and dt.date(query["start"]) == dt.date(query["end"]) then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	243 table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	244 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	245 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	246
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	247 prefix = url.build_path(prefix);
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	248 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, {
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	249 prefix = prefix;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	250 ["max-keys"] = query["max"] and tostring(query["max"]);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	251 }));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	252 if err or list_result.code ~= 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	253 return nil, err;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	254 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	255 local list_bucket_result = xml.parse(list_result.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	256 if list_bucket_result:get_child_text("IsTruncated") == "true" then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	257 local max_keys = list_bucket_result:get_child_text("MaxKeys");
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	258 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	259 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	260 local keys = array();
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	261 local iterwrap = function(...)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	262 return ...;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	263 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	264 if query["reverse"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	265 query["before"], query["after"] = query["after"], query["before"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	266 iterwrap = it.reverse;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	267 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	268 local found = not query["after"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	269 for content in iterwrap(list_bucket_result:childtags("Contents")) do
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	270 local key = url.parse_path(content:get_child_text("Key"));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	271 if found and query["before"] == key[6] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	272 break
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	273 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	274 if (not query["with"] or query["with"] == jid.unescape(key[5]))
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	275 and (not query["start"] or dt.date(query["start"]) >= key[6])
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	276 and (not query["end"] or dt.date(query["end"]) <= key[6])
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	277 and found then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	278 keys:push({ key = key[6]; date = key[5]; with = jid.unescape(key[4]) });
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	279 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	280 if not found and key[6] == query["after"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	281 found = not found
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	282 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	283 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	284 local i = 0;
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	285 local function get_next()
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	286 i = i + 1;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	287 local item = keys[i];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	288 if item == nil then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	289 return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	290 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	291 -- luacheck: ignore 431/err
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	292 local value, err = async.wait_for(new_request(self, "GET", self:_path(username or "@", item.date, nil, item.with, item.key)):next(on_result));
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	293 if not value then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	294 module:log("error", "%s", err);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	295 return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	296 end
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	297 local when = dt.parse(value:get_child_attr("delay", "urn:xmpp:delay", "stamp"));
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	298
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	299 if (not query["start"] or query["start"] >= when) and (not query["end"] or query["end"] <= when) then
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	300 return item.key, value.tags[2], when, item.with;
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	301 else
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	302 -- date was correct but not the time
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	303 return get_next();
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	304 end
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	305 end
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	306 return get_next;
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	307 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	308
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	309 function archive:users()
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	310 return it.unique(keyval.users(self));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	311 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	312
5697 4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	313 local function count(t) local n = 0; for _ in pairs(t) do n = n + 1; end return n; end
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	314
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	315 function archive:delete(username, query)
5697 4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	316 local deletions = {};
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	317 for key, _, when, with in self:find(username, query) do
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	318 deletions[key] = new_request(self, "DELETE", self:_path(username or "@", dt.date(when), nil, with, key));
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	319 end
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	320 return async.wait_for(promise.all(deletions):next(count));
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	321 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	322
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	323 module:provides("storage", driver);

Mercurial > prosody-modules

annotate mod_storage_s3/mod_storage_s3.lua @ 5715:8488ebde5739