# HG changeset patch # User tmolitor # Date 1493060243 -7200 # Node ID caabb980d1d829e2c1b1882b7f7534f766116bdd # Parent d96831e46b64237561109314ad632ba9747ffe3b# Parent 7a5dae85f26f0b5c9f2049d20d87e0b5b09f45e0 Merge commit diff -r d96831e46b64 -r caabb980d1d8 mod_conversejs/mod_conversejs.lua --- a/mod_conversejs/mod_conversejs.lua Mon Apr 24 20:56:56 2017 +0200 +++ b/mod_conversejs/mod_conversejs.lua Mon Apr 24 20:57:23 2017 +0200 @@ -26,6 +26,7 @@ bosh_service_url = module:http_url("bosh","/http-bind"); websocket_url = has_ws and module:http_url("websocket","xmpp-websocket"):gsub("^http", "ws") or nil; authentication = module:get_option_string("authentication") == "anonymous" and "anonymous" or "login"; + jid = module.host; })); end; } diff -r d96831e46b64 -r caabb980d1d8 mod_http_muc_log/mod_http_muc_log.lua --- a/mod_http_muc_log/mod_http_muc_log.lua Mon Apr 24 20:56:56 2017 +0200 +++ b/mod_http_muc_log/mod_http_muc_log.lua Mon Apr 24 20:57:23 2017 +0200 @@ -2,9 +2,7 @@ local datetime = require"util.datetime"; local jid_split = require"util.jid".split; local nodeprep = require"util.encodings".stringprep.nodeprep; -local uuid = require"util.uuid".generate; local it = require"util.iterators"; -local gettime = require"socket".gettime; local url = require"socket.url"; local os_time, os_date = os.time, os.date; local render = require"util.interpolation".new("%b{}", require"util.stanza".xml_escape); @@ -140,7 +138,10 @@ weeks[#weeks+1] = { days = days }; current_day = 1; end - days[current_day], current_day = { wday = tmp.wday, day = i, href = days_t[i] and datetime.date(days_t[i]) }, current_day+1; + days[current_day] = { + wday = tmp.wday, day = i, href = days_t[i] and datetime.date(days_t[i]) + }; + current_day = current_day+1; end end table.sort(year, sort_m); @@ -277,56 +278,10 @@ }); end -local cache = setmetatable({}, {__mode = 'v'}); - -local function with_cache(f) - return function (event, path) - local request, response = event.request, event.response; - local ckey = path or ""; - local cached = cache[ckey]; - - if cached then - local etag = cached.etag; - local if_none_match = request.headers.if_none_match; - if etag == if_none_match then - module:log("debug", "Client cache hit"); - return 304; - end - module:log("debug", "Server cache hit"); - response.headers.etag = etag; - response.headers.content_type = "text/html; charset=utf-8"; - return cached[1]; - end - - local start = gettime(); - local rendered = f(event, path); - module:log("debug", "Rendering took %dms", math.floor( (gettime() - start) * 1000 + 0.5)); - - if type(rendered) == "string" then - local etag = uuid(); - cached = { rendered, etag = etag, date = datetime.date() }; - response.headers.etag = etag; - cache[ckey] = cached; - end - - response.headers.content_type = "text/html; charset=utf-8"; - return rendered; - end -end - --- How is cache invalidation a hard problem? ;) -module:hook("muc-broadcast-message", function (event) - local room = event.room; - local room_name = jid_split(room.jid); - local today = datetime.date(); - cache[get_link(room_name)] = nil; - cache[get_link(room_name, today)] = nil; -end); - module:provides("http", { route = { ["GET /"] = list_rooms; - ["GET /*"] = with_cache(logs_page); + ["GET /*"] = logs_page; }; }); diff -r d96831e46b64 -r caabb980d1d8 mod_http_upload/README.markdown --- a/mod_http_upload/README.markdown Mon Apr 24 20:56:56 2017 +0200 +++ b/mod_http_upload/README.markdown Mon Apr 24 20:57:23 2017 +0200 @@ -27,6 +27,8 @@ Limits ------ +### Max size + A maximum file size can be set by: ``` {.lua} @@ -37,6 +39,30 @@ This can not be set over the value of `http_max_content_size` (default 10M). +### Max age + +Files can be set to be deleted after some time: + +``` lua +http_upload_expire_after = 60 * 60 * 24 * 7 -- a week in seconds +``` + +### User quota + +A total maximum size of all uploaded files per user can be set by: + +``` lua +http_upload_quota = 1234 -- bytes +``` + +### File types + +Accepted file types can be limited by MIME type: + +``` lua +http_upload_allowed_file_types = { "image/*", "text/plain" } +``` + Path ---- diff -r d96831e46b64 -r caabb980d1d8 mod_http_upload/mod_http_upload.lua --- a/mod_http_upload/mod_http_upload.lua Mon Apr 24 20:56:56 2017 +0200 +++ b/mod_http_upload/mod_http_upload.lua Mon Apr 24 20:57:23 2017 +0200 @@ -1,6 +1,6 @@ -- mod_http_upload -- --- Copyright (C) 2015 Kim Alvefur +-- Copyright (C) 2015-2017 Kim Alvefur -- -- This file is MIT/X11 licensed. -- @@ -13,6 +13,7 @@ local url = require "socket.url"; local dataform = require "util.dataforms".new; local datamanager = require "util.datamanager"; +local array = require "util.array"; local t_concat = table.concat; local t_insert = table.insert; local s_upper = string.upper; @@ -28,6 +29,9 @@ -- config local file_size_limit = module:get_option_number(module.name .. "_file_size_limit", 1024 * 1024); -- 1 MB +local quota = module:get_option_number(module.name .. "_quota"); +local max_age = module:get_option_number(module.name .. "_expire_after"); +local allowed_file_types = module:get_option_set(module.name .. "_allowed_file_types"); --- sanity local parser_body_limit = module:context("*"):get_option_number("http_max_content_size", 10*1024*1024); @@ -41,6 +45,9 @@ module:depends("http"); module:depends("disco"); +local http_files = module:depends("http_files"); +local mime_map = module:shared("/*/http_files/mime").types; + -- namespaces local namespace = "urn:xmpp:http:upload:0"; local legacy_namespace = "urn:xmpp:http:upload"; @@ -66,7 +73,45 @@ local storage_path = module:get_option_string(module.name .. "_path", join_path(prosody.paths.data, module.name)); lfs.mkdir(storage_path); -local function handle_request(origin, stanza, xmlns, filename, filesize) +local function expire(username, host) + if not max_age then return true; end + local uploads, err = datamanager.list_load(username, host, module.name); + if not uploads then return true; end + uploads = array(uploads); + local expiry = os.time() - max_age; + local upload_window = os.time() - 900; + uploads:filter(function (item) + local filename = item.filename; + if item.dir then + filename = join_path(storage_path, item.dir, item.filename); + end + if item.time < expiry then + local deleted, whynot = os.remove(filename); + if not deleted then + module:log("warn", "Could not delete expired upload %s: %s", filename, whynot or "delete failed"); + end + return false; + elseif item.time < upload_window and not lfs.attributes(filename) then + return false; -- File was not uploaded or has been deleted since + end + return true; + end); + return datamanager.list_store(username, host, module.name, uploads); +end + +local function check_quota(username, host, does_it_fit) + if not quota then return true; end + local uploads, err = datamanager.list_load(username, host, module.name); + if not uploads then return true; end + local sum = does_it_fit or 0; + for _, item in ipairs(uploads) do + sum = sum + item.size; + end + return sum < quota; +end + +local function handle_request(origin, stanza, xmlns, filename, filesize, mimetype) + local username, host = origin.username, origin.host; -- local clients only if origin.type ~= "c2s" then module:log("debug", "Request for upload slot from a %s", origin.type); @@ -79,6 +124,7 @@ origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid filename")); return true; end + expire(username, host); if not filesize then module:log("debug", "Missing file size"); origin.send(st.error_reply(stanza, "modify", "bad-request", "Missing or invalid file size")); @@ -89,7 +135,34 @@ :tag("file-too-large", {xmlns=xmlns}) :tag("max-file-size"):text(tostring(file_size_limit))); return true; + elseif not check_quota(username, host, filesize) then + module:log("debug", "Upload of %dB by %s would exceed quota", filesize, origin.full_jid); + origin.send(st.error_reply(stanza, "wait", "resource-constraint", "Quota reached")); + return true; end + + if mime_map then + local file_ext = filename:match("%.([^.]+)$"); + if not mimetype then + mimetype = "application/octet-stream"; + if file_ext then + mimetype = mime_map[file_ext] or mimetype; + end + else + if (not file_ext and mimetype ~= "application/octet-stream") or (file_ext and mime_map[file_ext] ~= mimetype) then + origin.send(st.error_reply(stanza, "modify", "bad-request", "MIME type does not match file extension")); + return true; + end + end + end + + if allowed_file_types then + if not (allowed_file_types:contains(mimetype) or allowed_file_types:contains(mimetype:gsub("/.*", "/*"))) then + origin.send(st.error_reply(stanza, "cancel", "not-allowed", "File type not allowed")); + return true; + end + end + local reply = st.reply(stanza); reply:tag("slot", { xmlns = xmlns }); @@ -98,10 +171,21 @@ until lfs.mkdir(join_path(storage_path, random_dir)) or not lfs.attributes(join_path(storage_path, random_dir, filename)) - datamanager.list_append(origin.username, origin.host, module.name, { - filename = join_path(storage_path, random_dir, filename), size = filesize, time = os.time() }); + local ok = datamanager.list_append(username, host, module.name, { + filename = filename, dir = random_dir, size = filesize, time = os.time() }); + + if not ok then + origin.send(st.error_reply(stanza, "wait", "internal-server-failure")); + return true; + end + local slot = random_dir.."/"..filename; pending_slots[slot] = origin.full_jid; + + module:add_timer(900, function() + pending_slots[slot] = nil; + end); + local base_url = module:http_url(); local slot_url = url.parse(base_url); slot_url.path = url.parse_path(slot_url.path or "/"); @@ -123,7 +207,8 @@ local request = stanza.tags[1]; local filename = request.attr.filename; local filesize = tonumber(request.attr.size); - return handle_request(origin, stanza, namespace, filename, filesize); + local mimetype = request.attr["content-type"]; + return handle_request(origin, stanza, namespace, filename, filesize, mimetype); end); module:hook("iq/host/"..legacy_namespace..":request", function (event) @@ -131,7 +216,8 @@ local request = stanza.tags[1]; local filename = request:get_child_text("filename"); local filesize = tonumber(request:get_child_text("size")); - return handle_request(origin, stanza, legacy_namespace, filename, filesize); + local mimetype = request:get_child_text("content-type"); + return handle_request(origin, stanza, legacy_namespace, filename, filesize, mimetype); end); -- http service @@ -217,7 +303,7 @@ end end -local serve_uploaded_files = module:depends("http_files").serve(storage_path); +local serve_uploaded_files = http_files.serve(storage_path); local function serve_head(event, path) event.response.send = send_response_sans_body; diff -r d96831e46b64 -r caabb980d1d8 mod_log_auth/mod_log_auth.lua --- a/mod_log_auth/mod_log_auth.lua Mon Apr 24 20:56:56 2017 +0200 +++ b/mod_log_auth/mod_log_auth.lua Mon Apr 24 20:57:23 2017 +0200 @@ -1,15 +1,19 @@ local mode = module:get_option_string("log_auth_ips", "failure"); -assert(({ all = true, failure = true, success = true })[mode], "Unknown log mode: "..tostring(mode).." - valid modes are 'all', 'failure', 'success'"); +assert(({ all = true, failure = true, success = true })[mode], + "Unknown log mode: "..tostring(mode).." - valid modes are 'all', 'failure', 'success'"); if mode == "failure" or mode == "all" then module:hook("authentication-failure", function (event) - module:log("info", "Failed authentication attempt (%s) for user %s from IP: %s", event.condition or "unknown-condition", event.session.username or "?", event.session.ip or "?"); + local session = event.session; + local username = session.username or session.sasl_handler and session.sasl_handler.username or "?"; + session.log("info", "Failed authentication attempt (%s) for user %s from IP: %s", + event.condition or "unknown-condition", username, session.ip or "?"); end); end if mode == "success" or mode == "all" then module:hook("authentication-success", function (event) local session = event.session; - module:log("info", "Successful authentication as %s from IP: %s", session.username, session.ip or "?"); + session.log("info", "Successful authentication as %s from IP: %s", session.username, session.ip or "?"); end); end diff -r d96831e46b64 -r caabb980d1d8 mod_log_http/README.markdown --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mod_log_http/README.markdown Mon Apr 24 20:57:23 2017 +0200 @@ -0,0 +1,22 @@ +--- +summary: HTTP request logging +... + +Introduction +============ + +This module logs *outgoing* requests that go via the internal net.http API. + +Output format liable to change. + +Configuration +============= + +One option is required, set `log_http_file` to the file path you would like to log to. + +Compatibility +============= + + ----- ------- + 0.10 Works (requires 375cf924fce1 or later) + ----- ------- diff -r d96831e46b64 -r caabb980d1d8 mod_log_http/mod_log_http.lua --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mod_log_http/mod_log_http.lua Mon Apr 24 20:57:23 2017 +0200 @@ -0,0 +1,78 @@ +module:set_global(); + +local http = require "net.http"; +local codes = require "net.http.codes"; +local json = require "util.json"; + +local log = assert(io.open(assert(module:get_option_string("log_http_file"), "Please supply log_http_file in the config"), "a+")); + +local function append_request(id, req) + local headers = {}; + for k, v in pairs(req.headers) do + table.insert(headers, { name = k, value = v }); + end + local queryString = {}; + if req.query then + for _, pair in ipairs(http.formdecode(req.query)) do + table.insert(queryString, pair); + end + end + log:write("<<<", json.encode({ + id = id; + type = "request"; + method = req.method; + url = req.url; + httpVersion = "HTTP/1.1"; + cookies = {}; + headers = headers; + queryString = queryString; + postData = req.body and { + mimeType = req.headers["Content-Type"]; + text = req.body; + } or nil; + headersSize = -1; + bodySize = -1; + }), "\n"); +end + +local function append_response(id, resp) + local headers = {}; + for k, v in pairs(resp.headers) do + table.insert(headers, { name = k, value = v }); + end + log:write(">>>", json.encode({ + id = id; + type = "response"; + status = resp.code; + statusText = codes[resp.code]; + httpVersion = resp.httpversion; + cookies = {}; + headers = headers; + content = resp.body and { + size = #resp.body; + mimeType = resp.headers.content_type; + text = resp.body; + } or nil; + headersSize = -1; + bodySize = -1; + }), "\n"); +end + +module:hook_object_event(http.events, "request", function (event) + module:log("warn", "Request to %s!", event.url); + append_request(event.request.id, event.request); +end); + +module:hook_object_event(http.events, "request-connection-error", function (event) + module:log("warn", "Failed to make request to %s!", event.url); +end); + +module:hook_object_event(http.events, "response", function (event) + module:log("warn", "Received response %d from %s!", event.code, event.url); + for k,v in pairs(event.response) do print("=====", k, v) end + append_response(event.request.id, event.response); +end); + +function module.unload() + log:close(); +end diff -r d96831e46b64 -r caabb980d1d8 mod_measure_storage/mod_measure_storage.lua --- a/mod_measure_storage/mod_measure_storage.lua Mon Apr 24 20:56:56 2017 +0200 +++ b/mod_measure_storage/mod_measure_storage.lua Mon Apr 24 20:57:23 2017 +0200 @@ -14,7 +14,7 @@ else metric_name = store_name.."_"..store_type.."_"..method_name; end - local measure_operation_started = module:measure(metric_name, metric_tags); + local measure_operation_started = module:measure(metric_name, "times", metric_tags); return function (...) module:log("debug", "Measuring storage operation %s (%s)", metric_name, metric_tags or "no tags"); @@ -43,7 +43,7 @@ local function hook_event(module) module:hook("store-opened", function(event) - event.store = wrap_store(module, event.store_name, event.store_type, event.store); + event.store = wrap_store(module, event.store_name, event.store_type or "keyval", event.store); end); end diff -r d96831e46b64 -r caabb980d1d8 mod_server_contact_info/mod_server_contact_info.lua --- a/mod_server_contact_info/mod_server_contact_info.lua Mon Apr 24 20:56:56 2017 +0200 +++ b/mod_server_contact_info/mod_server_contact_info.lua Mon Apr 24 20:57:23 2017 +0200 @@ -26,7 +26,7 @@ module:log("error", "No contact_info or admins set in config"); return -- Nothing to attach, so we'll just skip it. end - module:log("debug", "No contact_info in config, using admins as fallback"); + module:log("info", "No contact_info in config, using admins as fallback"); contact_config = { admin = array.collect( admins / function(admin) return "xmpp:" .. admin; end); }; diff -r d96831e46b64 -r caabb980d1d8 mod_storage_xmlarchive/mod_storage_xmlarchive.lua --- a/mod_storage_xmlarchive/mod_storage_xmlarchive.lua Mon Apr 24 20:56:56 2017 +0200 +++ b/mod_storage_xmlarchive/mod_storage_xmlarchive.lua Mon Apr 24 20:57:23 2017 +0200 @@ -91,7 +91,7 @@ local stream_session = { notopen = true }; local stream_callbacks = { handlestanza = cb, stream_ns = "jabber:client", default_ns = "jabber:client" }; local stream = new_stream(stream_session, stream_callbacks); - local dates = self:dates() or empty; + local dates = self:dates(username) or empty; local function reset_stream() stream:reset(); stream_session.notopen = true; @@ -279,14 +279,21 @@ end function provider:purge(username) - for store in dm.stores(username, module.host) do - local dates = dm.list_load(username, module.host, store) or empty; - if dates[1] and type(dates[1]) == "string" and dates[1]:match("^%d%d%d%d%-%d%d%-%d%d$") then - module:log("info", "Store %s looks like an archive store, emptying it...", store); - provider:open(store, "archive"):delete(username); + local encoded_username = dm.path_encode((username or "@") .. "@"); + local basepath = prosody.paths.data .. "/" .. dm.path_encode(module.host); + for store in lfs.dir(basepath) do + store = basepath .. "/" .. dm.path_encode(store); + if lfs.attributes(store, "mode") == "directory" then + for file in lfs.dir(store) do + if file:sub(1, #encoded_username) == encoded_username then + if file:sub(-4) == ".xml" or file:sub(-5) == ".list" then + os.remove(store .. "/" .. file); + end + end + end + return true; end end - return true; end module:provides("storage", provider);