Mercurial > libervia-backend

changeset 2370:2c2b826b0bb3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
plugin import: node can now be specified + added a "session" dict to keep import session data: import session data are data that can be used by importer to store anything which can be useful to keep between import methods.
author Goffi <goffi@goffi.org>
date Fri, 06 Oct 2017 08:52:51 +0200
parents cdaa58e14553
children 2268df8c99bf
files frontends/src/jp/cmd_blog.py src/plugins/plugin_blog_import.py src/plugins/plugin_import.py
diffstat 3 files changed, 26 insertions(+), 23 deletions(-) [+]
line wrap: on
line diff
--- a/frontends/src/jp/cmd_blog.py	Sun Oct 01 12:21:23 2017 +0200
+++ b/frontends/src/jp/cmd_blog.py	Fri Oct 06 08:52:51 2017 +0200
@@ -583,6 +583,8 @@
             help=_(u"importer specific options (see importer description)"))
         self.parser.add_argument('--service', type=base.unicode_decoder, default=u'', metavar=u'PUBSUB_SERVICE',
             help=_(u"PubSub service where the items must be uploaded (default: server)"))
+        self.parser.add_argument('-n', '--node', type=base.unicode_decoder, default=u'', metavar=u'PUBSUB_NODE',
+            help=_(u"PubSub node where the items must be uploaded (default: tickets' defaults)"))
         self.parser.add_argument("location", type=base.unicode_decoder, nargs='?',
             help=_(u"importer data location (see importer description), nothing to show importer description"))
 
@@ -641,7 +643,7 @@
                 options['upload_ignore_host'] = self.args.upload_ignore_host
             def gotId(id_):
                 self.progress_id = id_
-            self.host.bridge.blogImport(self.args.importer, self.args.location, options, self.args.service, self.profile,
+            self.host.bridge.blogImport(self.args.importer, self.args.location, options, self.args.service, self.args.node, self.profile,
                 callback=gotId, errback=self.error)
 
 
--- a/src/plugins/plugin_blog_import.py	Sun Oct 01 12:21:23 2017 +0200
+++ b/src/plugins/plugin_blog_import.py	Fri Oct 06 08:52:51 2017 +0200
@@ -27,7 +27,6 @@
 from twisted.words.xish import domish
 from sat.core import exceptions
 from sat.tools import xml_tools
-import collections
 import os
 import os.path
 import tempfile
@@ -53,9 +52,6 @@
 URL_REDIRECT_PREFIX = 'url_redirect_'
 
 
-BlogImporter = collections.namedtuple('BlogImporter', ('callback', 'short_desc', 'long_desc'))
-
-
 class BlogImportPlugin(object):
     BOOL_OPTIONS = (OPT_UPLOAD_IMAGES, OPT_IGNORE_TLS)
     OPT_DEFAULTS = {OPT_UPLOAD_IMAGES: True,
@@ -70,10 +66,10 @@
         self._s = self.host.plugins['TEXT-SYNTAXES']
         host.plugins['IMPORT'].initialize(self, u'blog')
 
-    def importItem(self, client, item_import_data, options, return_data, service, node):
+    def importItem(self, client, item_import_data, session, options, return_data, service, node):
         """importItem specialized for blog import
 
-        @param items_import_data(iterable[dict]):
+        @param item_import_data(dict):
             * mandatory keys:
                 'blog' (dict): microblog data of the blog post (cf. http://wiki.goffi.org/wiki/Bridge_API_-_Microblogging/en)
                     the importer MUST NOT create node or call XEP-0277 plugin itself
@@ -124,7 +120,7 @@
 
         return mb_data
 
-    def importSubItems(self, client, item_import_data, mb_data, options):
+    def importSubItems(self, client, item_import_data, mb_data, session, options):
         # comments data
         if len(item_import_data['comments']) != 1:
             raise NotImplementedError(u"can't manage multiple comment links")
@@ -143,12 +139,12 @@
                 raise exceptions.DataError(u"allow_comments set to False, but comments are there")
             return None
 
-    def publishItem(self, client, mb_data, service, node):
+    def publishItem(self, client, mb_data, service, node, session):
         log.debug(u"uploading item [{id}]: {title}".format(id=mb_data['id'], title=mb_data.get('title','')))
         return self._m.send(client, mb_data, service, node)
 
     @defer.inlineCallbacks
-    def itemFilters(self, client, mb_data, options):
+    def itemFilters(self, client, mb_data, session, options):
         """Apply filters according to options
 
         modify mb_data in place
--- a/src/plugins/plugin_import.py	Sun Oct 01 12:21:23 2017 +0200
+++ b/src/plugins/plugin_import.py	Fri Oct 06 08:52:51 2017 +0200
@@ -65,14 +65,14 @@
         import_handler.register = partial(self.register, import_handler)
         import_handler.unregister = partial(self.unregister, import_handler)
         import_handler.importers = {}
-        def _import(name, location, options, pubsub_service, profile):
-            return self._doImport(import_handler, name, location, options, pubsub_service, profile)
+        def _import(name, location, options, pubsub_service, pubsub_node, profile):
+            return self._doImport(import_handler, name, location, options, pubsub_service, pubsub_node, profile)
         def _importList():
             return self.listImporters(import_handler)
         def _importDesc(name):
             return self.getDescription(import_handler, name)
 
-        self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}ss', out_sign='s', method=_import, async=True)
+        self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}sss', out_sign='s', method=_import, async=True)
         self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList)
         self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc)
 
@@ -100,7 +100,7 @@
         else:
             return importer.short_desc, importer.long_desc
 
-    def _doImport(self, import_handler, name, location, options, pubsub_service='', profile=C.PROF_KEY_NONE):
+    def _doImport(self, import_handler, name, location, options, pubsub_service='', pubsub_node='', profile=C.PROF_KEY_NONE):
         client = self.host.getClient(profile)
         options = {key: unicode(value) for key, value in options.iteritems()}
         for option in import_handler.BOOL_OPTIONS:
@@ -108,10 +108,10 @@
                 options[option] = C.bool(options[option])
             except KeyError:
                 pass
-        return self.doImport(client, import_handler, unicode(name), unicode(location), options)
+        return self.doImport(client, import_handler, unicode(name), unicode(location), options, pubsub_service or None, pubsub_node or None)
 
     @defer.inlineCallbacks
-    def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None):
+    def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None, pubsub_node=None):
         """Import data
 
         @param import_handler(object): instance of the import handler
@@ -122,6 +122,8 @@
         @param options(dict, None): extra options.
         @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported
             None to use profile's server
+        @param pubsub_node(unicode, None): PubSub node to use
+            None to use importer's default node
         @return (unicode): progress id
         """
         if options is None:
@@ -157,17 +159,19 @@
                    }
         self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile)
         self.host.bridge.progressStarted(progress_id, metadata, client.profile)
-        url_redirect = {}
-        self.recursiveImport(client, import_handler, items_import_data, progress_id, options, url_redirect)
+        session = {}  # session data, can be use by importers
+        self.recursiveImport(client, import_handler, items_import_data, progress_id, session, options, None, pubsub_service, pubsub_node)
         defer.returnValue(progress_id)
 
     @defer.inlineCallbacks
-    def recursiveImport(self, client, import_handler, items_import_data, progress_id, options, return_data=None, service=None, node=None, depth=0):
+    def recursiveImport(self, client, import_handler, items_import_data, progress_id, session, options, return_data=None, service=None, node=None, depth=0):
         """Do the import recursively
 
         @param import_handler(object): instance of the import handler
         @param items_import_data(iterable): iterable of data as specified in [register]
         @param progress_id(unicode): id of progression
+        @param session(dict): data for this import session
+            can be used by importer so store any useful data
         @param options(dict): import options
         @param return_data(dict): data to return on progressFinished
         @param service(jid.JID, None): PubSub service to use
@@ -177,15 +181,16 @@
         if return_data is None:
             return_data = {}
         for idx, item_import_data in enumerate(items_import_data):
-            item_data = yield import_handler.importItem(client, item_import_data, options, return_data, service, node)
-            yield import_handler.itemFilters(client, item_data, options)
-            recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, options)
-            yield import_handler.publishItem(client, item_data, service, node)
+            item_data = yield import_handler.importItem(client, item_import_data, session, options, return_data, service, node)
+            yield import_handler.itemFilters(client, item_data, session, options)
+            recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, session, options)
+            yield import_handler.publishItem(client, item_data, service, node, session)
 
             if recurse_kwargs is not None:
                 recurse_kwargs['client'] = client
                 recurse_kwargs['import_handler'] = import_handler
                 recurse_kwargs['progress_id'] = progress_id
+                recurse_kwargs['session'] = session
                 recurse_kwargs.setdefault('options', options)
                 recurse_kwargs['return_data'] = return_data
                 recurse_kwargs['depth'] = depth + 1