# HG changeset patch # User Goffi # Date 1448210010 -3600 # Node ID d470affbe65c04eda93a908a65c9c9a282e80b08 # Parent baac2e1206002687609272f327b4726d6b0c2e0f plugin XEP-0363, upload: File upload (through HTTP upload only for now): - HTTP upload implementation - if the upload link is HTTPS, certificate is checked (can be disabled on demand) - file can be uploaded directly, or a put/get slot can be requested without actual upload. The later is mainly useful for distant frontends like Libervia - upload plugin manage different upload methods, in a similar way as file plugin - download url is sent in progressFinished metadata on successful upload diff -r baac2e120600 -r d470affbe65c src/bridge/bridge_constructor/bridge_template.ini --- a/src/bridge/bridge_constructor/bridge_template.ini Sun Nov 22 17:27:27 2015 +0100 +++ b/src/bridge/bridge_constructor/bridge_template.ini Sun Nov 22 17:33:30 2015 +0100 @@ -191,6 +191,7 @@ - hash_algo: alrorithm used to compute hash - hash_verified: C.BOOL_TRUE if hash is verified and OK C.BOOL_FALSE if hash was not received ([progressError] will be used if there is a mismatch) + - url: url linked to the progression (e.g. download url after a file upload) doc_param_2=%(doc_profile)s [progressError] diff -r baac2e120600 -r d470affbe65c src/plugins/plugin_misc_upload.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/plugin_misc_upload.py Sun Nov 22 17:33:30 2015 +0100 @@ -0,0 +1,123 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# SAT plugin for file tansfer +# Copyright (C) 2009, 2010, 2011, 2012, 2013, 2014, 2015 Jérôme Poisson (goffi@goffi.org) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from sat.core.i18n import _, D_ +from sat.core.constants import Const as C +from sat.core.log import getLogger +log = getLogger(__name__) +from sat.core import exceptions +from sat.tools import xml_tools +from twisted.internet import defer +from twisted.words.protocols.jabber import jid +import os +import os.path + + +PLUGIN_INFO = { + "name": "File Upload", + "import_name": "UPLOAD", + "type": C.PLUG_TYPE_MISC, + "main": "UploadPlugin", + "handler": "no", + "description": _("""File upload management""") +} + + +UPLOADING = D_(u'Please select a file to upload') +UPLOADING_TITLE = D_(u'File upload') +BOOL_OPTIONS = ('ignore-tls-errors',) + + +class UploadPlugin(object): + # TODO: plugin unload + + def __init__(self, host): + log.info(_("plugin Upload initialization")) + self.host = host + host.bridge.addMethod("fileUpload", ".plugin", in_sign='sssa{ss}s', out_sign='a{ss}', method=self._fileUpload, async=True) + self._upload_callbacks = [] + + def _fileUpload(self, filepath, filename, upload_jid_s='', options=None, profile=C.PROF_KEY_NONE): + upload_jid = jid.JID(upload_jid_s) if upload_jid_s else None + if options is None: + options = {} + # we convert values that are well-known booleans + for bool_option in BOOL_OPTIONS: + try: + options[bool_option] = C.bool(options[bool_option]) + except KeyError: + pass + + return self.fileUpload(filepath, filename or None, upload_jid, options or None, profile) + + @defer.inlineCallbacks + def fileUpload(self, filepath, filename, upload_jid, options, profile=C.PROF_KEY_NONE): + """Send a file using best available method + + @param filepath(str): absolute path to the file + @param filename(None, unicode): name to use for the upload + None to use basename of the path + @param upload_jid(jid.JID, None): upload capable entity jid, + or None to use autodetected, if possible + @param options(dict): option to use for the upload, may be: + - ignore-tls-errors(bool): True to ignore SSL/TLS certificate verification + used only if HTTPS transport is needed + @param profile: %(doc_profile)s + @return (dict): action dictionary, with progress id in case of success, else xmlui message + """ + if not os.path.isfile(filepath): + raise exceptions.DataError(u"The given path doesn't link to a file") + for method_name, available_cb, upload_cb, priority in self._upload_callbacks: + try: + upload_jid = yield available_cb(upload_jid, profile) + except exceptions.NotFound: + continue # no entity managing this extension found + log.info(u"{name} method will be used to upload the file".format(name=method_name)) + progress_id = yield defer.maybeDeferred(upload_cb, filepath, filename, upload_jid, options, profile) + defer.returnValue({'progress': progress_id}) + + # if we reach this point, no entity handling any known upload method has been found + msg = u"Can't find any method to upload a file" + log.warning(msg) + defer.returnValue({'xmlui': xml_tools.note(u"Can't upload file", msg, C.XMLUI_DATA_LVL_WARNING).toXml()}) + + def register(self, method_name, available_cb, upload_cb, priority=0): + """Register a fileUploading method + + @param method_name(unicode): short name for the method, must be unique + @param available_cb(callable): method to call to check if this method is usable + the callback must take two arguments: upload_jid (can be None) and profile + the callback must return the first entity found (being upload_jid or one of its components) + exceptions.NotFound must be raised if no entity has been found + @param upload_cb(callable): method to upload a file (must have the same signature as [fileUpload]) + @param priority(int): pririoty of this method, the higher available will be used + """ + assert method_name + for data in self._upload_callbacks: + if method_name == data[0]: + raise exceptions.ConflictError(u'A method with this name is already registered') + self._upload_callbacks.append((method_name, available_cb, upload_cb, priority)) + self._upload_callbacks.sort(key=lambda data: data[2], reverse=True) + + def unregister(self, method_name): + for idx, data in enumerate(self._upload_callbacks): + if data[0] == method_name: + del [idx] + return + raise exceptions.NotFound(u"The name to unregister doesn't exist") diff -r baac2e120600 -r d470affbe65c src/plugins/plugin_xep_0096.py --- a/src/plugins/plugin_xep_0096.py Sun Nov 22 17:27:27 2015 +0100 +++ b/src/plugins/plugin_xep_0096.py Sun Nov 22 17:33:30 2015 +0100 @@ -336,7 +336,7 @@ self.host.bridge.newAlert(_("The contact {} has refused your file").format(from_s), _("File refused"), "INFO", client.profile) else: log.warning(_(u"Error during file transfer")) - self.host.bridge.newAlert(_(u"Something went wrong during the file transfer session initialisation: {reason}").format(reason=unicode(stanza_err.value)), _("File transfer error"), "ERROR", client.profile) + self.host.bridge.newAlert(_(u"Something went wrong during the file transfer session initialisation: {reason}").format(reason=unicode(stanza_err)), _("File transfer error"), "ERROR", client.profile) elif failure.check(exceptions.DataError): log.warning(u'Invalid stanza received') else: diff -r baac2e120600 -r d470affbe65c src/plugins/plugin_xep_0363.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/plugin_xep_0363.py Sun Nov 22 17:33:30 2015 +0100 @@ -0,0 +1,285 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# SAT plugin for Jingle File Transfer (XEP-0363) +# Copyright (C) 2009, 2010, 2011, 2012, 2013, 2014, 2015 Jérôme Poisson (goffi@goffi.org) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from sat.core.i18n import _ +from sat.core.constants import Const as C +from sat.core.log import getLogger +log = getLogger(__name__) +from sat.core import exceptions +from wokkel import disco, iwokkel +from zope.interface import implements +from twisted.words.protocols.jabber import jid +from twisted.words.protocols.jabber.xmlstream import XMPPHandler +from twisted.internet import reactor +from twisted.internet import defer +from twisted.internet import ssl +from twisted.internet.interfaces import IOpenSSLClientConnectionCreator +from twisted.web import client as http_client +from twisted.web import http_headers +from twisted.web import iweb +from twisted.python import failure +from collections import namedtuple +from zope.interface import implementer +from OpenSSL import SSL +import os.path +import mimetypes + + +PLUGIN_INFO = { + "name": "HTTP File Upload", + "import_name": "XEP-0363", + "type": "XEP", + "protocols": ["XEP-0363"], + "dependencies": ["FILE", "UPLOAD"], + "main": "XEP_0363", + "handler": "yes", + "description": _("""Implementation of HTTP File Upload""") +} + +NS_HTTP_UPLOAD = 'urn:xmpp:http:upload' + + +Slot = namedtuple('Slot', ['put', 'get']) + + +@implementer(IOpenSSLClientConnectionCreator) +class NoCheckConnectionCreator(object): + + def __init__(self, hostname, ctx): + self._ctx = ctx + + def clientConnectionForTLS(self, tlsProtocol): + context = self._ctx + connection = SSL.Connection(context, None) + connection.set_app_data(tlsProtocol) + return connection + + +@implementer(iweb.IPolicyForHTTPS) +class NoCheckContextFactory(ssl.ClientContextFactory): + """Context factory which doesn't do TLS certificate check + + /!\\ it's obvisously a security flaw to use this class, + and it should be used only wiht explicite agreement from the end used + """ + + def creatorForNetloc(self, hostname, port): + log.warning(u"TLS check disabled for {host} on port {port}".format(host=hostname, port=port)) + certificateOptions = ssl.CertificateOptions(trustRoot=None) + return NoCheckConnectionCreator(hostname, certificateOptions.getContext()) + + +class XEP_0363(object): + + def __init__(self, host): + log.info(_("plugin HTTP File Upload initialization")) + self.host = host + host.bridge.addMethod("fileHTTPUpload", ".plugin", in_sign='sssbs', out_sign='', method=self._fileHTTPUpload) + host.bridge.addMethod("fileHTTPUploadGetSlot", ".plugin", in_sign='sisss', out_sign='(ss)', method=self._getSlot, async=True) + host.plugins['UPLOAD'].register(u"HTTP Upload", self.getHTTPUploadEntity, self.fileHTTPUpload) + + def getHandler(self, profile): + return XEP_0363_handler() + + @defer.inlineCallbacks + def getHTTPUploadEntity(self, upload_jid=None, profile=C.PROF_KEY_NONE): + """Get HTTP upload capable entity + + upload_jid is checked, then its components + @param upload_jid(None, jid.JID): entity to check + @return(D(jid.JID)): first HTTP upload capable entity + @raise exceptions.NotFound: no entity found + """ + client = self.host.getClient(profile) + try: + entity = client.http_upload_service + except AttributeError: + found_entities = yield self.host.findFeaturesSet((NS_HTTP_UPLOAD,), profile=client.profile) + try: + entity = client.http_upload_service = iter(found_entities).next() + except StopIteration: + entity = client.http_upload_service = None + + if entity is None: + raise failure.Failure(exceptions.NotFound(u'No HTTP upload entity found')) + + defer.returnValue(entity) + + def _fileHTTPUpload(self, filepath, filename='', upload_jid='', ignore_tls_errors=False, profile=C.PROF_KEY_NONE): + assert os.path.isabs(filepath) and os.path.isfile(filepath) + return self.fileHTTPUpload(filepath, filename or None, jid.JID(upload_jid) if upload_jid else None, {'ignore-tls-errors': ignore_tls_errors}, profile) + + def fileHTTPUpload(self, filepath, filename=None, upload_jid=None, options=None, profile=C.PROF_KEY_NONE): + """upload a file through HTTP + + @param filepath(str): absolute path of the file + @param filename(None, unicode): name to use for the upload + None to use basename of the path + @param upload_jid(jid.JID, None): upload capable entity jid, + or None to use autodetected, if possible + @param options(dict): options where key can be: + - ignore_tls_errors(bool): if True, SSL certificate will not be checked + @param profile: %(doc_profile)s + @return (D(unicode)): progress id + """ + if options is None: + options = {} + ignore_tls_errors = options.get('ignore-tls-errors', False) + client = self.host.getClient(profile) + filename = filename or os.path.basename(filepath) + size = os.path.getsize(filepath) + progress_id_d = defer.Deferred() + d = self.getSlot(client, filename, size, upload_jid=upload_jid) + d.addCallbacks(self._getSlotCb, self._getSlotEb, (client, progress_id_d, filepath, size, ignore_tls_errors), None, (client, progress_id_d)) + return progress_id_d + + def _getSlotEb(self, fail, client, progress_id_d): + """an error happened while trying to get slot""" + log.warning(u"Can't get upload slot: {reason}".format(reason=fail.value)) + progress_id_d.errback(fail) + + def _getSlotCb(self, slot, client, progress_id_d, path, size, ignore_tls_errors=False): + """Called when slot is received, try to do the upload + + @param slot(Slot): slot instance with the get and put urls + @param progress_id_d(defer.Deferred): Deferred to call when progress_id is known + @param path(str): path to the file to upload + @param size(int): size of the file to upload + @param ignore_tls_errors(bool): ignore TLS certificate is True + @return (tuple + """ + log.debug(u"Got upload slot: {}".format(slot)) + sat_file = self.host.plugins['FILE'].File(self.host, path, size=size, auto_end_signals=False, profile=client.profile) + progress_id_d.callback(sat_file.uid) + file_producer = http_client.FileBodyProducer(sat_file) + if ignore_tls_errors: + agent = http_client.Agent(reactor, NoCheckContextFactory()) + else: + agent = http_client.Agent(reactor) + d = agent.request('PUT', slot.put.encode('utf-8'), http_headers.Headers({'User-Agent': [C.APP_NAME.encode('utf-8')]}), file_producer) + d.addCallbacks(self._uploadCb, self._uploadEb, (sat_file, slot), None, (sat_file,)) + return d + + def _uploadCb(self, dummy, sat_file, slot): + """Called once file is successfully uploaded + + @param sat_file(SatFile): file used for the upload + should be closed, be is needed to send the progressFinished signal + @param slot(Slot): put/get urls + """ + log.info(u"HTTP upload finished") + sat_file.progressFinished({'url': slot.get}) + + def _uploadEb(self, fail, sat_file): + """Called on unsuccessful upload + + @param sat_file(SatFile): file used for the upload + should be closed, be is needed to send the progressError signal + """ + try: + wrapped_fail = fail.value.reasons[0] + except (AttributeError, IndexError): + sat_file.progressError(unicode(fail)) + raise fail + else: + if wrapped_fail.check(SSL.Error): + msg = u"TLS validation error, can't connect to HTTPS server" + log.warning(msg + ": " + unicode(wrapped_fail.value)) + sat_file.progressError(msg) + + def _gotSlot(self, iq_elt, client): + """Slot have been received + + This method convert the iq_elt result to a Slot instance + @param iq_elt(domish.Element): result as specified in XEP-0363 + """ + try: + slot_elt = iq_elt.elements(NS_HTTP_UPLOAD, 'slot').next() + put_url = unicode(slot_elt.elements(NS_HTTP_UPLOAD, 'put').next()) + get_url = unicode(slot_elt.elements(NS_HTTP_UPLOAD, 'get').next()) + except StopIteration: + raise exceptions.DataError(u"Incorrect stanza received from server") + slot = Slot(put=put_url, get=get_url) + return slot + + def _getSlot(self, filename, size, content_type, upload_jid, profile_key=C.PROF_KEY_NONE): + """Get a upload slot + + This method can be used when uploading is done by the frontend + @param filename(unicode): name of the file to upload + @param size(int): size of the file (must be non null) + @param upload_jid(jid.JID(), None, ''): HTTP upload capable entity + @param content_type(unicode, None): MIME type of the content + empty string or None to guess automatically + """ + filename.replace('/', '_') + client = self.host.getClient(profile_key) + return self.getSlot(client, filename, size, content_type or None, upload_jid or None) + + def getSlot(self, client, filename, size, content_type=None, upload_jid=None): + """Get a slot (i.e. download/upload links) + + @param filename(unicode): name to use for the upload + @param size(int): size of the file to upload (must be >0) + @param content_type(None, unicode): MIME type of the content + None to autodetect + @param upload_jid(jid.JID, None): HTTP upload capable upload_jid + or None to use the server component (if any) + @param client: %(doc_client)s + @return (Slot): the upload (put) and download (get) URLs + @raise exceptions.NotFound: no HTTP upload capable upload_jid has been found + """ + assert filename and size + if content_type is None: + # TODO: manage python magic for file guessing (in a dedicated plugin ?) + content_type = mimetypes.guess_type(filename, strict=False)[0] + + if upload_jid is None: + try: + upload_jid = client.http_upload_service + except AttributeError: + d = self.getHTTPUploadEntity(profile=client.profile) + d.addCallback(lambda found_entity: self.getSlot(client, filename, size, content_type, found_entity)) + return d + else: + if upload_jid is None: + raise failure.Failure(exceptions.NotFound(u'No HTTP upload entity found')) + + iq_elt = client.IQ('get') + iq_elt['to'] = upload_jid.full() + request_elt = iq_elt.addElement((NS_HTTP_UPLOAD, 'request')) + request_elt.addElement('filename', content=filename) + request_elt.addElement('size', content=unicode(size)) + if content_type is not None: + request_elt.addElement('content-type', content=content_type) + + d = iq_elt.send() + d.addCallback(self._gotSlot, client) + + return d + + +class XEP_0363_handler(XMPPHandler): + implements(iwokkel.IDisco) + + def getDiscoInfo(self, requestor, target, nodeIdentifier=''): + return [disco.DiscoFeature(NS_HTTP_UPLOAD)] + + def getDiscoItems(self, requestor, target, nodeIdentifier=''): + return []