prosody-modules: mod_storage_s3/mod_storage

annotate mod_storage_s3/mod_storage_s3.lua @ 5787:e79f9dec35c0

mod_c2s_conn_throttle: Reduce log level from error->info Our general policy is that "error" should never be triggerable by remote entities, and that it is always about something that requires admin intervention. This satisfies neither condition. The "warn" level can be used for unexpected events/behaviour triggered by remote entities, and this could qualify. However I don't think failed auth attempts are unexpected enough. I selected "info" because it is what is also used for other notable session lifecycle events.

author	Matthew Wild <mwild1@gmail.com>
date	Thu, 07 Dec 2023 15:46:50 +0000
parents	ff8b6d0b3bfa
children	f89367e421d1

rev	line source
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	1 local http = require "prosody.net.http";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	2 local array = require "prosody.util.array";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	3 local async = require "prosody.util.async";
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	4 local dt = require "prosody.util.datetime";
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	5 local hashes = require "prosody.util.hashes";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	6 local httputil = require "prosody.util.http";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	7 local it = require "prosody.util.iterators";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	8 local jid = require "prosody.util.jid";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	9 local json = require "prosody.util.json";
5697 4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	10 local promise = require "prosody.util.promise";
5733 b6518a71ca7e mod_storage_s3: Implement search for set of IDs Kim Alvefur <zash@zash.se> parents: 5732 diff changeset	11 local set = require "prosody.util.set";
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	12 local st = require "prosody.util.stanza";
5698 991fb904fa49 mod_storage_s3: Sort imports Kim Alvefur <zash@zash.se> parents: 5697 diff changeset	13 local uuid = require "prosody.util.uuid";
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	14 local xml = require "prosody.util.xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	15 local url = require "socket.url";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	16
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	17 local new_uuid = uuid.v7 or uuid.generate;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	18 local hmac_sha256 = hashes.hmac_sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	19 local sha256 = hashes.sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	20
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	21 local driver = {};
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	22
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	23 local bucket = module:get_option_string("s3_bucket", "prosody");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	24 local base_uri = module:get_option_string("s3_base_uri", "http://localhost:9000");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	25 local region = module:get_option_string("s3_region", "us-east-1");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	26
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	27 local access_key = module:get_option_string("s3_access_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	28 local secret_key = module:get_option_string("s3_secret_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	29
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	30 local aws4_format = "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	31
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	32 local function aws_auth(event)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	33 local request, options = event.request, event.options;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	34 local method = options.method or "GET";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	35 local query = options.query;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	36 local payload = options.body;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	37
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	38 local payload_type = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	39 if st.is_stanza(payload) then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	40 payload_type = "application/xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	41 payload = tostring(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	42 elseif payload ~= nil then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	43 payload_type = "application/json";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	44 payload = json.encode(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	45 end
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	46 options.body = payload;
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	47
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	48 local payload_hash = sha256(payload or "", true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	49
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	50 local now = os.time();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	51 local aws_datetime = os.date("!%Y%m%dT%H%M%SZ", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	52 local aws_date = os.date("!%Y%m%d", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	53
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	54 local headers = {
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	55 ["Accept"] = "/";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	56 ["Authorization"] = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	57 ["Content-Type"] = payload_type;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	58 ["Host"] = request.authority;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	59 ["User-Agent"] = "Prosody XMPP Server";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	60 ["X-Amz-Content-Sha256"] = payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	61 ["X-Amz-Date"] = aws_datetime;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	62 };
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	63
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	64 local canonical_uri = url.build({ path = request.path });
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	65 local canonical_query = "";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	66 local canonical_headers = array();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	67 local signed_headers = array()
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	68
5670 2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	69 if query then
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	70 local sorted_query = array();
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	71 for name, value in it.sorted_pairs(query) do
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	72 sorted_query:push({ name = name; value = value });
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	73 end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	74 sorted_query:sort(function (a,b) return a.name < b.name end)
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	75 canonical_query = httputil.formencode(sorted_query):gsub("%%%x%x", string.upper);
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	76 request.query = canonical_query;
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	77 end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part Kim Alvefur <zash@zash.se> parents: 5669 diff changeset	78
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	79 for header_name, header_value in it.sorted_pairs(headers) do
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	80 header_name = header_name:lower();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	81 canonical_headers:push(header_name .. ":" .. header_value .. "\n");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	82 signed_headers:push(header_name);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	83 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	84
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	85 canonical_headers = canonical_headers:concat();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	86 signed_headers = signed_headers:concat(";");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	87
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	88 local scope = aws_date .. "/" .. region .. "/s3/aws4_request";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	89
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	90 local canonical_request = method .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	91 .. canonical_uri .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	92 .. canonical_query .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	93 .. canonical_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	94 .. signed_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	95 .. payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	96
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	97 local signature_payload = "AWS4-HMAC-SHA256" .. "\n" .. aws_datetime .. "\n" .. scope .. "\n" .. sha256(canonical_request, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	98
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	99 -- This can be cached?
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	100 local date_key = hmac_sha256("AWS4" .. secret_key, aws_date);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	101 local date_region_key = hmac_sha256(date_key, region);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	102 local date_region_service_key = hmac_sha256(date_region_key, "s3");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	103 local signing_key = hmac_sha256(date_region_service_key, "aws4_request");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	104
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	105 local signature = hmac_sha256(signing_key, signature_payload, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	106
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	107 headers["Authorization"] = string.format(aws4_format, access_key, scope, signed_headers, signature);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	108
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	109 options.headers = headers;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	110 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	111
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	112 function driver:open(store, typ)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	113 local mt = self[typ or "keyval"]
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	114 if not mt then
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	115 return nil, "unsupported-store";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	116 end
5700 9de7a1b36efb mod_storage_s3: Enable connection pooling added in latest trunk Kim Alvefur <zash@zash.se> parents: 5699 diff changeset	117 local httpclient = http.new({ connection_pooling = true });
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	118 httpclient.events.add_handler("pre-request", aws_auth);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	119 return setmetatable({ store = store; bucket = bucket; type = typ; http = httpclient }, mt);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	120 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	121
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	122 local keyval = { };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	123 driver.keyval = { __index = keyval; __name = module.name .. " keyval store" };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	124
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	125 local function new_request(self, method, path, query, payload)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	126 local request = url.parse(base_uri);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	127 request.path = path;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	128
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	129 return self.http:request(url.build(request), { method = method; body = payload; query = query });
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	130 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	131
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	132 -- coerce result back into Prosody data type
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	133 local function on_result(response)
5676 799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	134 if response.code == 404 and response.request.method == "GET" then
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	135 return nil;
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data Kim Alvefur <zash@zash.se> parents: 5675 diff changeset	136 end
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	137 if response.code >= 400 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	138 error(response.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	139 end
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	140 local content_type = response.headers["content-type"];
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	141 if content_type == "application/json" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	142 return json.decode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	143 elseif content_type == "application/xml" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	144 return xml.parse(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	145 elseif content_type == "application/x-www-form-urlencoded" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	146 return httputil.formdecode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	147 else
5699 83a2fb6df746 mod_storage_s3: Fix logging Kim Alvefur <zash@zash.se> parents: 5698 diff changeset	148 module:log("warn", "Unknown response data type %s", content_type);
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	149 return response.body;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	150 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	151 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	152
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	153 function keyval:_path(key)
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	154 return url.build_path({
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	155 is_absolute = true;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	156 bucket;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	157 jid.escape(module.host);
5736 ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	158 jid.escape(key or "@");
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	159 jid.escape(self.store);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	160 })
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	161 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	162
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	163 function keyval:get(user)
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	164 return async.wait_for(new_request(self, "GET", self:_path(user)):next(on_result));
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	165 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	166
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	167 function keyval:set(user, data)
5671 c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	168
c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	169 if data == nil or (type(data) == "table" and next(data) == nil) then
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	170 return async.wait_for(new_request(self, "DELETE", self:_path(user)));
5671 c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	171 end
c8322c64a548 mod_storage_s3: Implement keyvalue deletion Kim Alvefur <zash@zash.se> parents: 5670 diff changeset	172
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	173 return async.wait_for(new_request(self, "PUT", self:_path(user), nil, data));
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	174 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	175
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	176 function keyval:users()
5672 c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	177 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
5758 32bc648e3892 mod_storage_s3: Fix passing of prefixes, should not be urlencoded Kim Alvefur <zash@zash.se> parents: 5737 diff changeset	178 local prefix = jid.escape(module.host) .. "/";
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	179 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, { prefix = prefix }))
5672 c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	180 if err or list_result.code ~= 200 then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	181 return nil, err;
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	182 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	183 local list_bucket_result = xml.parse(list_result.body);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	184 if list_bucket_result:get_child_text("IsTruncated") == "true" then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	185 local max_keys = list_bucket_result:get_child_text("MaxKeys");
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	186 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	187 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	188 local keys = array();
5736 ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	189 local store_part = jid.escape(self.store);
5672 c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	190 for content in list_bucket_result:childtags("Contents") do
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	191 local key = url.parse_path(content:get_child_text("Key"));
5736 ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	192 if key[3] == store_part then
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	193 keys:push(jid.unescape(key[2]));
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	194 end
5672 c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	195 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	196 return function()
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	197 return keys:pop();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually) Kim Alvefur <zash@zash.se> parents: 5671 diff changeset	198 end
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	199 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	200
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	201 local archive = {};
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	202 driver.archive = { __index = archive };
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	203
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	204 archive.caps = {
5732 e8938a3166d2 mod_storage_s3: Advertise full id range archive query capability Kim Alvefur <zash@zash.se> parents: 5728 diff changeset	205 full_id_range = true; -- both before and after used
5733 b6518a71ca7e mod_storage_s3: Implement search for set of IDs Kim Alvefur <zash@zash.se> parents: 5732 diff changeset	206 ids = true;
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	207 };
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	208
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	209 function archive:_path(username, date, when, with, key)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	210 return url.build_path({
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	211 is_absolute = true;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	212 bucket;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	213 jid.escape(module.host);
5736 ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	214 jid.escape(username or "@");
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	215 jid.escape(self.store);
5736 ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	216 date or dt.date(when);
5728 80702e33ba71 mod_storage_s3: Fix storing archives for host itself (e.g. mod_audit) Kim Alvefur <zash@zash.se> parents: 5700 diff changeset	217 jid.escape(with and jid.prep(with) or "@");
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	218 key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	219 })
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	220 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	221
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	222
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	223 -- PUT .../with/when/id
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	224 function archive:append(username, key, value, when, with)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	225 key = key or new_uuid();
5760 59e38aaa3ec1 mod_storage_s3: Remove wrapper and original timestamp from payload (BC) Kim Alvefur <zash@zash.se> parents: 5759 diff changeset	226 return async.wait_for(new_request(self, "PUT", self:_path(username, nil, when, with, key), nil, value):next(function(r)
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	227 if r.code == 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	228 return key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	229 else
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	230 error(r.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	231 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	232 end));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	233 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	234
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	235 function archive:find(username, query)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	236 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
5758 32bc648e3892 mod_storage_s3: Fix passing of prefixes, should not be urlencoded Kim Alvefur <zash@zash.se> parents: 5737 diff changeset	237 local prefix = { jid.escape(module.host); jid.escape(username or "@"); jid.escape(self.store) };
5674 51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	238 if not query then
51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	239 query = {};
51d0311747fa mod_storage_s3: Handle archive query without parameters Kim Alvefur <zash@zash.se> parents: 5673 diff changeset	240 end
5736 ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	241
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	242 if query["start"] and query["end"] and dt.date(query["start"]) == dt.date(query["end"]) then
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	243 table.insert(prefix, dt.date(query["start"]));
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	244 if query["with"] then
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	245 table.insert(prefix, jid.escape(query["with"]));
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	246 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	247 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	248
5758 32bc648e3892 mod_storage_s3: Fix passing of prefixes, should not be urlencoded Kim Alvefur <zash@zash.se> parents: 5737 diff changeset	249 prefix = table.concat(prefix, "/").."/";
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	250 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, {
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	251 prefix = prefix;
5762 ff8b6d0b3bfa mod_storage_s3: Fix mapping archive query limit to ?max-keys= Kim Alvefur <zash@zash.se> parents: 5760 diff changeset	252 ["max-keys"] = query["limit"] and tostring(query["limit"]);
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	253 }));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	254 if err or list_result.code ~= 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	255 return nil, err;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	256 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	257 local list_bucket_result = xml.parse(list_result.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	258 if list_bucket_result:get_child_text("IsTruncated") == "true" then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	259 local max_keys = list_bucket_result:get_child_text("MaxKeys");
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	260 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	261 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	262 local keys = array();
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	263 local iterwrap = function(...)
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	264 return ...;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	265 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	266 if query["reverse"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	267 query["before"], query["after"] = query["after"], query["before"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	268 iterwrap = it.reverse;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	269 end
5733 b6518a71ca7e mod_storage_s3: Implement search for set of IDs Kim Alvefur <zash@zash.se> parents: 5732 diff changeset	270 local ids = query["ids"] and set.new(query["ids"]);
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	271 local found = not query["after"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	272 for content in iterwrap(list_bucket_result:childtags("Contents")) do
5737 72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	273 local date, with, id = table.unpack(url.parse_path(content:get_child_text("Key")), 4);
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	274 local when = dt.parse(content:get_child_text("LastModified"));
5736 ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	275 with = jid.unescape(with);
ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	276 if found and query["before"] == id then
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	277 break
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	278 end
5736 ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	279 if (not query["with"] or query["with"] == with)
5737 72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	280 and (not query["start"] or query["start"] <= when)
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	281 and (not query["end"] or query["end"] >= when)
5736 ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	282 and (not ids or ids:contains(id))
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	283 and found then
5737 72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	284 keys:push({ key = id; date = date; when = when; with = with });
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	285 end
5736 ba731ff5b895 mod_storage_s3: Reorder path components (BC: invalidates any existing data) Kim Alvefur <zash@zash.se> parents: 5735 diff changeset	286 if not found and id == query["after"] then
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	287 found = not found
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	288 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	289 end
5737 72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	290 keys:sort(function(a, b)
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	291 if a.date ~= b.date then
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	292 return a.date < b.date
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	293 end
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	294 if a.when ~= b.when then
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	295 return a.when < b.when;
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	296 end
5759 c7affea8bb24 mod_storage_s3: Fix sorting items by correct field Kim Alvefur <zash@zash.se> parents: 5758 diff changeset	297 return a.key < b.key;
5737 72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	298 end);
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	299 if query["reverse"] then
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	300 keys:reverse();
72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	301 end
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	302 local i = 0;
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	303 local function get_next()
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	304 i = i + 1;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	305 local item = keys[i];
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	306 if item == nil then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	307 return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	308 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	309 -- luacheck: ignore 431/err
5695 b4632d5f840b mod_storage_s3: Move request signing into a net.http hook Kim Alvefur <zash@zash.se> parents: 5676 diff changeset	310 local value, err = async.wait_for(new_request(self, "GET", self:_path(username or "@", item.date, nil, item.with, item.key)):next(on_result));
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	311 if not value then
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	312 module:log("error", "%s", err);
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	313 return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	314 end
5737 72b0fa7e36dc mod_storage_s3: Sort archive items by LastModified Kim Alvefur <zash@zash.se> parents: 5736 diff changeset	315 return item.key, value, item.when, item.with;
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	316 end
5696 66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime Kim Alvefur <zash@zash.se> parents: 5695 diff changeset	317 return get_next;
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	318 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	319
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	320 function archive:users()
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	321 return it.unique(keyval.users(self));
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	322 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	323
5697 4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	324 local function count(t) local n = 0; for _ in pairs(t) do n = n + 1; end return n; end
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	325
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	326 function archive:delete(username, query)
5697 4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	327 local deletions = {};
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	328 for key, _, when, with in self:find(username, query) do
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	329 deletions[key] = new_request(self, "DELETE", self:_path(username or "@", dt.date(when), nil, with, key));
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	330 end
4a0279c5c7ed mod_storage_s3: Implement archive store deletion Kim Alvefur <zash@zash.se> parents: 5696 diff changeset	331 return async.wait_for(promise.all(deletions):next(count));
5673 b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	332 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage Kim Alvefur <zash@zash.se> parents: 5672 diff changeset	333
5669 30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver Kim Alvefur <zash@zash.se> parents: diff changeset	334 module:provides("storage", driver);

Mercurial > prosody-modules

annotate mod_storage_s3/mod_storage_s3.lua @ 5787:e79f9dec35c0