Mercurial > prosody-modules
annotate mod_storage_s3/mod_storage_s3.lua @ 5715:8488ebde5739
mod_http_oauth2: Skip consent screen if requested by client and same scopes already granted
This follows the intent behind the OpenID Connect 'prompt' parameter
when it does not include the 'consent' keyword, that is the client
wishes to skip the consent screen. If the user has already granted the
exact same scopes to the exact same client in the past, then one can
assume that they may grant it again.
author | Kim Alvefur <zash@zash.se> |
---|---|
date | Tue, 14 Nov 2023 23:03:37 +0100 |
parents | 9de7a1b36efb |
children | 80702e33ba71 |
rev | line source |
---|---|
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
1 local http = require "prosody.net.http"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
2 local array = require "prosody.util.array"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
3 local async = require "prosody.util.async"; |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
4 local dt = require "prosody.util.datetime"; |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
5 local hashes = require "prosody.util.hashes"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
6 local httputil = require "prosody.util.http"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
7 local it = require "prosody.util.iterators"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
8 local jid = require "prosody.util.jid"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
9 local json = require "prosody.util.json"; |
5697
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5696
diff
changeset
|
10 local promise = require "prosody.util.promise"; |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
11 local st = require "prosody.util.stanza"; |
5698 | 12 local uuid = require "prosody.util.uuid"; |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
13 local xml = require "prosody.util.xml"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
14 local url = require "socket.url"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
15 |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
16 local new_uuid = uuid.v7 or uuid.generate; |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
17 local hmac_sha256 = hashes.hmac_sha256; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
18 local sha256 = hashes.sha256; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
19 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
20 local driver = {}; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
21 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
22 local bucket = module:get_option_string("s3_bucket", "prosody"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
23 local base_uri = module:get_option_string("s3_base_uri", "http://localhost:9000"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
24 local region = module:get_option_string("s3_region", "us-east-1"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
25 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
26 local access_key = module:get_option_string("s3_access_key"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
27 local secret_key = module:get_option_string("s3_secret_key"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
28 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
29 local aws4_format = "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
30 |
5695
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
31 local function aws_auth(event) |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
32 local request, options = event.request, event.options; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
33 local method = options.method or "GET"; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
34 local query = options.query; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
35 local payload = options.body; |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
36 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
37 local payload_type = nil; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
38 if st.is_stanza(payload) then |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
39 payload_type = "application/xml"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
40 payload = tostring(payload); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
41 elseif payload ~= nil then |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
42 payload_type = "application/json"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
43 payload = json.encode(payload); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
44 end |
5695
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
45 options.body = payload; |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
46 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
47 local payload_hash = sha256(payload or "", true); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
48 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
49 local now = os.time(); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
50 local aws_datetime = os.date("!%Y%m%dT%H%M%SZ", now); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
51 local aws_date = os.date("!%Y%m%d", now); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
52 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
53 local headers = { |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
54 ["Accept"] = "*/*"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
55 ["Authorization"] = nil; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
56 ["Content-Type"] = payload_type; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
57 ["Host"] = request.authority; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
58 ["User-Agent"] = "Prosody XMPP Server"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
59 ["X-Amz-Content-Sha256"] = payload_hash; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
60 ["X-Amz-Date"] = aws_datetime; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
61 }; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
62 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
63 local canonical_uri = url.build({ path = request.path }); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
64 local canonical_query = ""; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
65 local canonical_headers = array(); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
66 local signed_headers = array() |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
67 |
5670
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5669
diff
changeset
|
68 if query then |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5669
diff
changeset
|
69 local sorted_query = array(); |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5669
diff
changeset
|
70 for name, value in it.sorted_pairs(query) do |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5669
diff
changeset
|
71 sorted_query:push({ name = name; value = value }); |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5669
diff
changeset
|
72 end |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5669
diff
changeset
|
73 sorted_query:sort(function (a,b) return a.name < b.name end) |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5669
diff
changeset
|
74 canonical_query = httputil.formencode(sorted_query):gsub("%%%x%x", string.upper); |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5669
diff
changeset
|
75 request.query = canonical_query; |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5669
diff
changeset
|
76 end |
2c9d72ef829e
mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents:
5669
diff
changeset
|
77 |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
78 for header_name, header_value in it.sorted_pairs(headers) do |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
79 header_name = header_name:lower(); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
80 canonical_headers:push(header_name .. ":" .. header_value .. "\n"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
81 signed_headers:push(header_name); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
82 end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
83 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
84 canonical_headers = canonical_headers:concat(); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
85 signed_headers = signed_headers:concat(";"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
86 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
87 local scope = aws_date .. "/" .. region .. "/s3/aws4_request"; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
88 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
89 local canonical_request = method .. "\n" |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
90 .. canonical_uri .. "\n" |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
91 .. canonical_query .. "\n" |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
92 .. canonical_headers .. "\n" |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
93 .. signed_headers .. "\n" |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
94 .. payload_hash; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
95 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
96 local signature_payload = "AWS4-HMAC-SHA256" .. "\n" .. aws_datetime .. "\n" .. scope .. "\n" .. sha256(canonical_request, true); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
97 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
98 -- This can be cached? |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
99 local date_key = hmac_sha256("AWS4" .. secret_key, aws_date); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
100 local date_region_key = hmac_sha256(date_key, region); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
101 local date_region_service_key = hmac_sha256(date_region_key, "s3"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
102 local signing_key = hmac_sha256(date_region_service_key, "aws4_request"); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
103 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
104 local signature = hmac_sha256(signing_key, signature_payload, true); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
105 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
106 headers["Authorization"] = string.format(aws4_format, access_key, scope, signed_headers, signature); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
107 |
5695
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
108 options.headers = headers; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
109 end |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
110 |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
111 function driver:open(store, typ) |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
112 local mt = self[typ or "keyval"] |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
113 if not mt then |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
114 return nil, "unsupported-store"; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
115 end |
5700
9de7a1b36efb
mod_storage_s3: Enable connection pooling added in latest trunk
Kim Alvefur <zash@zash.se>
parents:
5699
diff
changeset
|
116 local httpclient = http.new({ connection_pooling = true }); |
5695
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
117 httpclient.events.add_handler("pre-request", aws_auth); |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
118 return setmetatable({ store = store; bucket = bucket; type = typ; http = httpclient }, mt); |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
119 end |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
120 |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
121 local keyval = { }; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
122 driver.keyval = { __index = keyval; __name = module.name .. " keyval store" }; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
123 |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
124 local function new_request(self, method, path, query, payload) |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
125 local request = url.parse(base_uri); |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
126 request.path = path; |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
127 |
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
128 return self.http:request(url.build(request), { method = method; body = payload; query = query }); |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
129 end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
130 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
131 -- coerce result back into Prosody data type |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
132 local function on_result(response) |
5676
799f69a5921a
mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
133 if response.code == 404 and response.request.method == "GET" then |
799f69a5921a
mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
134 return nil; |
799f69a5921a
mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents:
5675
diff
changeset
|
135 end |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
136 if response.code >= 400 then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
137 error(response.body); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
138 end |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
139 local content_type = response.headers["content-type"]; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
140 if content_type == "application/json" then |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
141 return json.decode(response.body); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
142 elseif content_type == "application/xml" then |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
143 return xml.parse(response.body); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
144 elseif content_type == "application/x-www-form-urlencoded" then |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
145 return httputil.formdecode(response.body); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
146 else |
5699 | 147 module:log("warn", "Unknown response data type %s", content_type); |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
148 return response.body; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
149 end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
150 end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
151 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
152 function keyval:_path(key) |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
153 return url.build_path({ |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
154 is_absolute = true; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
155 bucket; |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
156 jid.escape(module.host); |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
157 jid.escape(self.store); |
5675
17ea26cf7259
mod_storage_s3: Use '@' as placeholder for empty (host) store slots
Kim Alvefur <zash@zash.se>
parents:
5674
diff
changeset
|
158 jid.escape(key or "@"); |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
159 }) |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
160 end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
161 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
162 function keyval:get(user) |
5695
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
163 return async.wait_for(new_request(self, "GET", self:_path(user)):next(on_result)); |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
164 end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
165 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
166 function keyval:set(user, data) |
5671
c8322c64a548
mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents:
5670
diff
changeset
|
167 |
c8322c64a548
mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents:
5670
diff
changeset
|
168 if data == nil or (type(data) == "table" and next(data) == nil) then |
5695
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
169 return async.wait_for(new_request(self, "DELETE", self:_path(user))); |
5671
c8322c64a548
mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents:
5670
diff
changeset
|
170 end |
c8322c64a548
mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents:
5670
diff
changeset
|
171 |
5695
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
172 return async.wait_for(new_request(self, "PUT", self:_path(user), nil, data)); |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
173 end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
174 |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
175 function keyval:users() |
5672
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
176 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true }); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
177 local prefix = url.build_path({ jid.escape(module.host); jid.escape(self.store); is_directory = true }); |
5695
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
178 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, { prefix = prefix })) |
5672
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
179 if err or list_result.code ~= 200 then |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
180 return nil, err; |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
181 end |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
182 local list_bucket_result = xml.parse(list_result.body); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
183 if list_bucket_result:get_child_text("IsTruncated") == "true" then |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
184 local max_keys = list_bucket_result:get_child_text("MaxKeys"); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
185 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
186 end |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
187 local keys = array(); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
188 for content in list_bucket_result:childtags("Contents") do |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
189 local key = url.parse_path(content:get_child_text("Key")); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
190 keys:push(jid.unescape(key[3])); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
191 end |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
192 return function() |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
193 return keys:pop(); |
c74a96dc5d58
mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents:
5671
diff
changeset
|
194 end |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
195 end |
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
196 |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
197 local archive = {}; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
198 driver.archive = { __index = archive }; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
199 |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
200 archive.caps = { |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
201 }; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
202 |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
203 function archive:_path(username, date, when, with, key) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
204 return url.build_path({ |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
205 is_absolute = true; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
206 bucket; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
207 jid.escape(module.host); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
208 jid.escape(self.store); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
209 jid.escape(username); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
210 jid.escape(jid.prep(with)); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
211 date or dt.date(when); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
212 key; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
213 }) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
214 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
215 |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
216 |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
217 -- PUT .../with/when/id |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
218 function archive:append(username, key, value, when, with) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
219 local wrapper = st.stanza("wrapper"); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
220 -- Minio had trouble with timestamps, probably the ':' characters, in paths. |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
221 wrapper:tag("delay", { xmlns = "urn:xmpp:delay"; stamp = dt.datetime(when) }):up(); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
222 wrapper:add_direct_child(value); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
223 key = key or new_uuid(); |
5695
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
224 return async.wait_for(new_request(self, "PUT", self:_path(username, nil, when, with, key), nil, wrapper):next(function(r) |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
225 if r.code == 200 then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
226 return key; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
227 else |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
228 error(r.body); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
229 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
230 end)); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
231 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
232 |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
233 function archive:find(username, query) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
234 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true }); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
235 local prefix = { jid.escape(module.host); jid.escape(self.store); is_directory = true }; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
236 table.insert(prefix, jid.escape(username or "@")); |
5674
51d0311747fa
mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
237 if not query then |
51d0311747fa
mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
238 query = {}; |
51d0311747fa
mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents:
5673
diff
changeset
|
239 end |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
240 if query["with"] then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
241 table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24)); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
242 if query["start"] and query["end"] and dt.date(query["start"]) == dt.date(query["end"]) then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
243 table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24)); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
244 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
245 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
246 |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
247 prefix = url.build_path(prefix); |
5695
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
248 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, { |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
249 prefix = prefix; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
250 ["max-keys"] = query["max"] and tostring(query["max"]); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
251 })); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
252 if err or list_result.code ~= 200 then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
253 return nil, err; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
254 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
255 local list_bucket_result = xml.parse(list_result.body); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
256 if list_bucket_result:get_child_text("IsTruncated") == "true" then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
257 local max_keys = list_bucket_result:get_child_text("MaxKeys"); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
258 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
259 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
260 local keys = array(); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
261 local iterwrap = function(...) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
262 return ...; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
263 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
264 if query["reverse"] then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
265 query["before"], query["after"] = query["after"], query["before"]; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
266 iterwrap = it.reverse; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
267 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
268 local found = not query["after"]; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
269 for content in iterwrap(list_bucket_result:childtags("Contents")) do |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
270 local key = url.parse_path(content:get_child_text("Key")); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
271 if found and query["before"] == key[6] then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
272 break |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
273 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
274 if (not query["with"] or query["with"] == jid.unescape(key[5])) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
275 and (not query["start"] or dt.date(query["start"]) >= key[6]) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
276 and (not query["end"] or dt.date(query["end"]) <= key[6]) |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
277 and found then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
278 keys:push({ key = key[6]; date = key[5]; with = jid.unescape(key[4]) }); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
279 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
280 if not found and key[6] == query["after"] then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
281 found = not found |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
282 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
283 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
284 local i = 0; |
5696
66986f5271c3
mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents:
5695
diff
changeset
|
285 local function get_next() |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
286 i = i + 1; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
287 local item = keys[i]; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
288 if item == nil then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
289 return nil; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
290 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
291 -- luacheck: ignore 431/err |
5695
b4632d5f840b
mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents:
5676
diff
changeset
|
292 local value, err = async.wait_for(new_request(self, "GET", self:_path(username or "@", item.date, nil, item.with, item.key)):next(on_result)); |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
293 if not value then |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
294 module:log("error", "%s", err); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
295 return nil; |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
296 end |
5696
66986f5271c3
mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents:
5695
diff
changeset
|
297 local when = dt.parse(value:get_child_attr("delay", "urn:xmpp:delay", "stamp")); |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
298 |
5696
66986f5271c3
mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents:
5695
diff
changeset
|
299 if (not query["start"] or query["start"] >= when) and (not query["end"] or query["end"] <= when) then |
66986f5271c3
mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents:
5695
diff
changeset
|
300 return item.key, value.tags[2], when, item.with; |
66986f5271c3
mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents:
5695
diff
changeset
|
301 else |
66986f5271c3
mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents:
5695
diff
changeset
|
302 -- date was correct but not the time |
66986f5271c3
mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents:
5695
diff
changeset
|
303 return get_next(); |
66986f5271c3
mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents:
5695
diff
changeset
|
304 end |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
305 end |
5696
66986f5271c3
mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents:
5695
diff
changeset
|
306 return get_next; |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
307 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
308 |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
309 function archive:users() |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
310 return it.unique(keyval.users(self)); |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
311 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
312 |
5697
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5696
diff
changeset
|
313 local function count(t) local n = 0; for _ in pairs(t) do n = n + 1; end return n; end |
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5696
diff
changeset
|
314 |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
315 function archive:delete(username, query) |
5697
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5696
diff
changeset
|
316 local deletions = {}; |
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5696
diff
changeset
|
317 for key, _, when, with in self:find(username, query) do |
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5696
diff
changeset
|
318 deletions[key] = new_request(self, "DELETE", self:_path(username or "@", dt.date(when), nil, with, key)); |
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5696
diff
changeset
|
319 end |
4a0279c5c7ed
mod_storage_s3: Implement archive store deletion
Kim Alvefur <zash@zash.se>
parents:
5696
diff
changeset
|
320 return async.wait_for(promise.all(deletions):next(count)); |
5673
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
321 end |
b17ba149b7c5
mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents:
5672
diff
changeset
|
322 |
5669
30f91daa40b4
mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
323 module:provides("storage", driver); |