changeset 1640:d470affbe65c

plugin XEP-0363, upload: File upload (through HTTP upload only for now): - HTTP upload implementation - if the upload link is HTTPS, certificate is checked (can be disabled on demand) - file can be uploaded directly, or a put/get slot can be requested without actual upload. The later is mainly useful for distant frontends like Libervia - upload plugin manage different upload methods, in a similar way as file plugin - download url is sent in progressFinished metadata on successful upload
author Goffi <goffi@goffi.org>
date Sun, 22 Nov 2015 17:33:30 +0100
parents baac2e120600
children 44a14f83e64b
files src/bridge/bridge_constructor/bridge_template.ini src/plugins/plugin_misc_upload.py src/plugins/plugin_xep_0096.py src/plugins/plugin_xep_0363.py
diffstat 4 files changed, 410 insertions(+), 1 deletions(-) [+]
line wrap: on
line diff
--- a/src/bridge/bridge_constructor/bridge_template.ini	Sun Nov 22 17:27:27 2015 +0100
+++ b/src/bridge/bridge_constructor/bridge_template.ini	Sun Nov 22 17:33:30 2015 +0100
@@ -191,6 +191,7 @@
     - hash_algo: alrorithm used to compute hash
     - hash_verified: C.BOOL_TRUE if hash is verified and OK
         C.BOOL_FALSE if hash was not received ([progressError] will be used if there is a mismatch)
+    - url: url linked to the progression (e.g. download url after a file upload)
 doc_param_2=%(doc_profile)s
 
 [progressError]
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/plugin_misc_upload.py	Sun Nov 22 17:33:30 2015 +0100
@@ -0,0 +1,123 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# SAT plugin for file tansfer
+# Copyright (C) 2009, 2010, 2011, 2012, 2013, 2014, 2015 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from sat.core.i18n import _, D_
+from sat.core.constants import Const as C
+from sat.core.log import getLogger
+log = getLogger(__name__)
+from sat.core import exceptions
+from sat.tools import xml_tools
+from twisted.internet import defer
+from twisted.words.protocols.jabber import jid
+import os
+import os.path
+
+
+PLUGIN_INFO = {
+    "name": "File Upload",
+    "import_name": "UPLOAD",
+    "type": C.PLUG_TYPE_MISC,
+    "main": "UploadPlugin",
+    "handler": "no",
+    "description": _("""File upload management""")
+}
+
+
+UPLOADING = D_(u'Please select a file to upload')
+UPLOADING_TITLE = D_(u'File upload')
+BOOL_OPTIONS = ('ignore-tls-errors',)
+
+
+class UploadPlugin(object):
+    # TODO: plugin unload
+
+    def __init__(self, host):
+        log.info(_("plugin Upload initialization"))
+        self.host = host
+        host.bridge.addMethod("fileUpload", ".plugin", in_sign='sssa{ss}s', out_sign='a{ss}', method=self._fileUpload, async=True)
+        self._upload_callbacks = []
+
+    def _fileUpload(self, filepath, filename, upload_jid_s='', options=None, profile=C.PROF_KEY_NONE):
+        upload_jid = jid.JID(upload_jid_s) if upload_jid_s else None
+        if options is None:
+            options = {}
+        # we convert values that are well-known booleans
+        for bool_option in BOOL_OPTIONS:
+            try:
+                options[bool_option] = C.bool(options[bool_option])
+            except KeyError:
+                pass
+
+        return self.fileUpload(filepath, filename or None, upload_jid, options or None, profile)
+
+    @defer.inlineCallbacks
+    def fileUpload(self, filepath, filename, upload_jid, options, profile=C.PROF_KEY_NONE):
+        """Send a file using best available method
+
+        @param filepath(str): absolute path to the file
+        @param filename(None, unicode): name to use for the upload
+            None to use basename of the path
+        @param upload_jid(jid.JID, None): upload capable entity jid,
+            or None to use autodetected, if possible
+        @param options(dict): option to use for the upload, may be:
+            - ignore-tls-errors(bool): True to ignore SSL/TLS certificate verification
+                used only if HTTPS transport is needed
+        @param profile: %(doc_profile)s
+        @return (dict): action dictionary, with progress id in case of success, else xmlui message
+        """
+        if not os.path.isfile(filepath):
+            raise exceptions.DataError(u"The given path doesn't link to a file")
+        for method_name, available_cb, upload_cb, priority in self._upload_callbacks:
+            try:
+                upload_jid = yield available_cb(upload_jid, profile)
+            except exceptions.NotFound:
+                continue # no entity managing this extension found
+            log.info(u"{name} method will be used to upload the file".format(name=method_name))
+            progress_id = yield defer.maybeDeferred(upload_cb, filepath, filename, upload_jid, options, profile)
+            defer.returnValue({'progress': progress_id})
+
+        # if we reach this point, no entity handling any known upload method has been found
+        msg = u"Can't find any method to upload a file"
+        log.warning(msg)
+        defer.returnValue({'xmlui': xml_tools.note(u"Can't upload file", msg, C.XMLUI_DATA_LVL_WARNING).toXml()})
+
+    def register(self, method_name, available_cb, upload_cb, priority=0):
+        """Register a fileUploading method
+
+        @param method_name(unicode): short name for the method, must be unique
+        @param available_cb(callable): method to call to check if this method is usable
+           the callback must take two arguments: upload_jid (can be None) and profile
+           the callback must return the first entity found (being upload_jid or one of its components)
+           exceptions.NotFound must be raised if no entity has been found
+        @param upload_cb(callable): method to upload a file (must have the same signature as [fileUpload])
+        @param priority(int): pririoty of this method, the higher available will be used
+        """
+        assert method_name
+        for data in self._upload_callbacks:
+            if method_name == data[0]:
+                raise exceptions.ConflictError(u'A method with this name is already registered')
+        self._upload_callbacks.append((method_name, available_cb, upload_cb, priority))
+        self._upload_callbacks.sort(key=lambda data: data[2], reverse=True)
+
+    def unregister(self, method_name):
+        for idx, data in enumerate(self._upload_callbacks):
+            if data[0] == method_name:
+                del [idx]
+                return
+        raise exceptions.NotFound(u"The name to unregister doesn't exist")
--- a/src/plugins/plugin_xep_0096.py	Sun Nov 22 17:27:27 2015 +0100
+++ b/src/plugins/plugin_xep_0096.py	Sun Nov 22 17:33:30 2015 +0100
@@ -336,7 +336,7 @@
                 self.host.bridge.newAlert(_("The contact {} has refused your file").format(from_s), _("File refused"), "INFO", client.profile)
             else:
                 log.warning(_(u"Error during file transfer"))
-                self.host.bridge.newAlert(_(u"Something went wrong during the file transfer session initialisation: {reason}").format(reason=unicode(stanza_err.value)), _("File transfer error"), "ERROR", client.profile)
+                self.host.bridge.newAlert(_(u"Something went wrong during the file transfer session initialisation: {reason}").format(reason=unicode(stanza_err)), _("File transfer error"), "ERROR", client.profile)
         elif failure.check(exceptions.DataError):
             log.warning(u'Invalid stanza received')
         else:
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/plugin_xep_0363.py	Sun Nov 22 17:33:30 2015 +0100
@@ -0,0 +1,285 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# SAT plugin for Jingle File Transfer (XEP-0363)
+# Copyright (C) 2009, 2010, 2011, 2012, 2013, 2014, 2015 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from sat.core.i18n import _
+from sat.core.constants import Const as C
+from sat.core.log import getLogger
+log = getLogger(__name__)
+from sat.core import exceptions
+from wokkel import disco, iwokkel
+from zope.interface import implements
+from twisted.words.protocols.jabber import jid
+from twisted.words.protocols.jabber.xmlstream import XMPPHandler
+from twisted.internet import reactor
+from twisted.internet import defer
+from twisted.internet import ssl
+from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
+from twisted.web import client as http_client
+from twisted.web import http_headers
+from twisted.web import iweb
+from twisted.python import failure
+from collections import namedtuple
+from zope.interface import implementer
+from OpenSSL import SSL
+import os.path
+import mimetypes
+
+
+PLUGIN_INFO = {
+    "name": "HTTP File Upload",
+    "import_name": "XEP-0363",
+    "type": "XEP",
+    "protocols": ["XEP-0363"],
+    "dependencies": ["FILE", "UPLOAD"],
+    "main": "XEP_0363",
+    "handler": "yes",
+    "description": _("""Implementation of HTTP File Upload""")
+}
+
+NS_HTTP_UPLOAD = 'urn:xmpp:http:upload'
+
+
+Slot = namedtuple('Slot', ['put', 'get'])
+
+
+@implementer(IOpenSSLClientConnectionCreator)
+class NoCheckConnectionCreator(object):
+
+    def __init__(self, hostname, ctx):
+        self._ctx = ctx
+
+    def clientConnectionForTLS(self, tlsProtocol):
+        context = self._ctx
+        connection = SSL.Connection(context, None)
+        connection.set_app_data(tlsProtocol)
+        return connection
+
+
+@implementer(iweb.IPolicyForHTTPS)
+class NoCheckContextFactory(ssl.ClientContextFactory):
+    """Context factory which doesn't do TLS certificate check
+
+    /!\\ it's obvisously a security flaw to use this class,
+    and it should be used only wiht explicite agreement from the end used
+    """
+
+    def creatorForNetloc(self, hostname, port):
+        log.warning(u"TLS check disabled for {host} on port {port}".format(host=hostname, port=port))
+        certificateOptions = ssl.CertificateOptions(trustRoot=None)
+        return NoCheckConnectionCreator(hostname, certificateOptions.getContext())
+
+
+class XEP_0363(object):
+
+    def __init__(self, host):
+        log.info(_("plugin HTTP File Upload initialization"))
+        self.host = host
+        host.bridge.addMethod("fileHTTPUpload", ".plugin", in_sign='sssbs', out_sign='', method=self._fileHTTPUpload)
+        host.bridge.addMethod("fileHTTPUploadGetSlot", ".plugin", in_sign='sisss', out_sign='(ss)', method=self._getSlot, async=True)
+        host.plugins['UPLOAD'].register(u"HTTP Upload", self.getHTTPUploadEntity, self.fileHTTPUpload)
+
+    def getHandler(self, profile):
+        return XEP_0363_handler()
+
+    @defer.inlineCallbacks
+    def getHTTPUploadEntity(self, upload_jid=None, profile=C.PROF_KEY_NONE):
+         """Get HTTP upload capable entity
+
+         upload_jid is checked, then its components
+         @param upload_jid(None, jid.JID): entity to check
+         @return(D(jid.JID)): first HTTP upload capable entity
+         @raise exceptions.NotFound: no entity found
+         """
+         client = self.host.getClient(profile)
+         try:
+             entity = client.http_upload_service
+         except AttributeError:
+             found_entities = yield self.host.findFeaturesSet((NS_HTTP_UPLOAD,), profile=client.profile)
+             try:
+                 entity = client.http_upload_service = iter(found_entities).next()
+             except StopIteration:
+                 entity = client.http_upload_service = None
+
+         if entity is None:
+             raise failure.Failure(exceptions.NotFound(u'No HTTP upload entity found'))
+
+         defer.returnValue(entity)
+
+    def _fileHTTPUpload(self, filepath, filename='', upload_jid='', ignore_tls_errors=False, profile=C.PROF_KEY_NONE):
+        assert os.path.isabs(filepath) and os.path.isfile(filepath)
+        return self.fileHTTPUpload(filepath, filename or None, jid.JID(upload_jid) if upload_jid else None, {'ignore-tls-errors': ignore_tls_errors}, profile)
+
+    def fileHTTPUpload(self, filepath, filename=None, upload_jid=None, options=None, profile=C.PROF_KEY_NONE):
+        """upload a file through HTTP
+
+        @param filepath(str): absolute path of the file
+        @param filename(None, unicode): name to use for the upload
+            None to use basename of the path
+        @param upload_jid(jid.JID, None): upload capable entity jid,
+            or None to use autodetected, if possible
+        @param options(dict): options where key can be:
+            - ignore_tls_errors(bool): if True, SSL certificate will not be checked
+        @param profile: %(doc_profile)s
+        @return (D(unicode)): progress id
+        """
+        if options is None:
+            options = {}
+        ignore_tls_errors = options.get('ignore-tls-errors', False)
+        client = self.host.getClient(profile)
+        filename = filename or os.path.basename(filepath)
+        size = os.path.getsize(filepath)
+        progress_id_d = defer.Deferred()
+        d = self.getSlot(client, filename, size, upload_jid=upload_jid)
+        d.addCallbacks(self._getSlotCb, self._getSlotEb, (client, progress_id_d, filepath, size, ignore_tls_errors), None, (client, progress_id_d))
+        return progress_id_d
+
+    def _getSlotEb(self, fail, client, progress_id_d):
+        """an error happened while trying to get slot"""
+        log.warning(u"Can't get upload slot: {reason}".format(reason=fail.value))
+        progress_id_d.errback(fail)
+
+    def _getSlotCb(self, slot, client, progress_id_d, path, size, ignore_tls_errors=False):
+        """Called when slot is received, try to do the upload
+
+        @param slot(Slot): slot instance with the get and put urls
+        @param progress_id_d(defer.Deferred): Deferred to call when progress_id is known
+        @param path(str): path to the file to upload
+        @param size(int): size of the file to upload
+        @param ignore_tls_errors(bool): ignore TLS certificate is True
+        @return (tuple
+        """
+        log.debug(u"Got upload slot: {}".format(slot))
+        sat_file = self.host.plugins['FILE'].File(self.host, path, size=size, auto_end_signals=False, profile=client.profile)
+        progress_id_d.callback(sat_file.uid)
+        file_producer = http_client.FileBodyProducer(sat_file)
+        if ignore_tls_errors:
+            agent = http_client.Agent(reactor, NoCheckContextFactory())
+        else:
+            agent = http_client.Agent(reactor)
+        d = agent.request('PUT', slot.put.encode('utf-8'), http_headers.Headers({'User-Agent': [C.APP_NAME.encode('utf-8')]}), file_producer)
+        d.addCallbacks(self._uploadCb, self._uploadEb, (sat_file, slot), None, (sat_file,))
+        return d
+
+    def _uploadCb(self, dummy, sat_file, slot):
+        """Called once file is successfully uploaded
+
+        @param sat_file(SatFile): file used for the upload
+            should be closed, be is needed to send the progressFinished signal
+        @param slot(Slot): put/get urls
+        """
+        log.info(u"HTTP upload finished")
+        sat_file.progressFinished({'url': slot.get})
+
+    def _uploadEb(self, fail, sat_file):
+        """Called on unsuccessful upload
+
+        @param sat_file(SatFile): file used for the upload
+            should be closed, be is needed to send the progressError signal
+        """
+        try:
+            wrapped_fail = fail.value.reasons[0]
+        except (AttributeError, IndexError):
+            sat_file.progressError(unicode(fail))
+            raise fail
+        else:
+            if wrapped_fail.check(SSL.Error):
+                msg = u"TLS validation error, can't connect to HTTPS server"
+                log.warning(msg + ": " + unicode(wrapped_fail.value))
+                sat_file.progressError(msg)
+
+    def _gotSlot(self, iq_elt, client):
+        """Slot have been received
+
+        This method convert the iq_elt result to a Slot instance
+        @param iq_elt(domish.Element): <IQ/> result as specified in XEP-0363
+        """
+        try:
+            slot_elt = iq_elt.elements(NS_HTTP_UPLOAD, 'slot').next()
+            put_url = unicode(slot_elt.elements(NS_HTTP_UPLOAD, 'put').next())
+            get_url = unicode(slot_elt.elements(NS_HTTP_UPLOAD, 'get').next())
+        except StopIteration:
+            raise exceptions.DataError(u"Incorrect stanza received from server")
+        slot = Slot(put=put_url, get=get_url)
+        return slot
+
+    def _getSlot(self, filename, size, content_type, upload_jid, profile_key=C.PROF_KEY_NONE):
+        """Get a upload slot
+
+        This method can be used when uploading is done by the frontend
+        @param filename(unicode): name of the file to upload
+        @param size(int): size of the file (must be non null)
+        @param upload_jid(jid.JID(), None, ''): HTTP upload capable entity
+        @param content_type(unicode, None): MIME type of the content
+            empty string or None to guess automatically
+        """
+        filename.replace('/', '_')
+        client = self.host.getClient(profile_key)
+        return  self.getSlot(client, filename, size, content_type or None, upload_jid or None)
+
+    def getSlot(self, client, filename, size, content_type=None, upload_jid=None):
+        """Get a slot (i.e. download/upload links)
+
+        @param filename(unicode): name to use for the upload
+        @param size(int): size of the file to upload (must be >0)
+        @param content_type(None, unicode): MIME type of the content
+            None to autodetect
+        @param upload_jid(jid.JID, None): HTTP upload capable upload_jid
+            or None to use the server component (if any)
+        @param client: %(doc_client)s
+        @return (Slot): the upload (put) and download (get) URLs
+        @raise exceptions.NotFound: no HTTP upload capable upload_jid has been found
+        """
+        assert filename and size
+        if content_type is None:
+            # TODO: manage python magic for file guessing (in a dedicated plugin ?)
+            content_type = mimetypes.guess_type(filename, strict=False)[0]
+
+        if upload_jid is None:
+            try:
+                upload_jid = client.http_upload_service
+            except AttributeError:
+                d = self.getHTTPUploadEntity(profile=client.profile)
+                d.addCallback(lambda found_entity: self.getSlot(client, filename, size, content_type, found_entity))
+                return d
+            else:
+                if upload_jid is None:
+                    raise failure.Failure(exceptions.NotFound(u'No HTTP upload entity found'))
+
+        iq_elt = client.IQ('get')
+        iq_elt['to'] = upload_jid.full()
+        request_elt = iq_elt.addElement((NS_HTTP_UPLOAD, 'request'))
+        request_elt.addElement('filename', content=filename)
+        request_elt.addElement('size', content=unicode(size))
+        if content_type is not None:
+            request_elt.addElement('content-type', content=content_type)
+
+        d = iq_elt.send()
+        d.addCallback(self._gotSlot, client)
+
+        return d
+
+
+class XEP_0363_handler(XMPPHandler):
+    implements(iwokkel.IDisco)
+
+    def getDiscoInfo(self, requestor, target, nodeIdentifier=''):
+        return [disco.DiscoFeature(NS_HTTP_UPLOAD)]
+
+    def getDiscoItems(self, requestor, target, nodeIdentifier=''):
+        return []