# HG changeset patch # User Goffi # Date 1507272771 -7200 # Node ID 2c2b826b0bb35ff122f628ca25cd14e7f43ca032 # Parent cdaa58e1455347fd04345657d5b3b7c70bc5f7fe plugin import: node can now be specified + added a "session" dict to keep import session data: import session data are data that can be used by importer to store anything which can be useful to keep between import methods. diff -r cdaa58e14553 -r 2c2b826b0bb3 frontends/src/jp/cmd_blog.py --- a/frontends/src/jp/cmd_blog.py Sun Oct 01 12:21:23 2017 +0200 +++ b/frontends/src/jp/cmd_blog.py Fri Oct 06 08:52:51 2017 +0200 @@ -583,6 +583,8 @@ help=_(u"importer specific options (see importer description)")) self.parser.add_argument('--service', type=base.unicode_decoder, default=u'', metavar=u'PUBSUB_SERVICE', help=_(u"PubSub service where the items must be uploaded (default: server)")) + self.parser.add_argument('-n', '--node', type=base.unicode_decoder, default=u'', metavar=u'PUBSUB_NODE', + help=_(u"PubSub node where the items must be uploaded (default: tickets' defaults)")) self.parser.add_argument("location", type=base.unicode_decoder, nargs='?', help=_(u"importer data location (see importer description), nothing to show importer description")) @@ -641,7 +643,7 @@ options['upload_ignore_host'] = self.args.upload_ignore_host def gotId(id_): self.progress_id = id_ - self.host.bridge.blogImport(self.args.importer, self.args.location, options, self.args.service, self.profile, + self.host.bridge.blogImport(self.args.importer, self.args.location, options, self.args.service, self.args.node, self.profile, callback=gotId, errback=self.error) diff -r cdaa58e14553 -r 2c2b826b0bb3 src/plugins/plugin_blog_import.py --- a/src/plugins/plugin_blog_import.py Sun Oct 01 12:21:23 2017 +0200 +++ b/src/plugins/plugin_blog_import.py Fri Oct 06 08:52:51 2017 +0200 @@ -27,7 +27,6 @@ from twisted.words.xish import domish from sat.core import exceptions from sat.tools import xml_tools -import collections import os import os.path import tempfile @@ -53,9 +52,6 @@ URL_REDIRECT_PREFIX = 'url_redirect_' -BlogImporter = collections.namedtuple('BlogImporter', ('callback', 'short_desc', 'long_desc')) - - class BlogImportPlugin(object): BOOL_OPTIONS = (OPT_UPLOAD_IMAGES, OPT_IGNORE_TLS) OPT_DEFAULTS = {OPT_UPLOAD_IMAGES: True, @@ -70,10 +66,10 @@ self._s = self.host.plugins['TEXT-SYNTAXES'] host.plugins['IMPORT'].initialize(self, u'blog') - def importItem(self, client, item_import_data, options, return_data, service, node): + def importItem(self, client, item_import_data, session, options, return_data, service, node): """importItem specialized for blog import - @param items_import_data(iterable[dict]): + @param item_import_data(dict): * mandatory keys: 'blog' (dict): microblog data of the blog post (cf. http://wiki.goffi.org/wiki/Bridge_API_-_Microblogging/en) the importer MUST NOT create node or call XEP-0277 plugin itself @@ -124,7 +120,7 @@ return mb_data - def importSubItems(self, client, item_import_data, mb_data, options): + def importSubItems(self, client, item_import_data, mb_data, session, options): # comments data if len(item_import_data['comments']) != 1: raise NotImplementedError(u"can't manage multiple comment links") @@ -143,12 +139,12 @@ raise exceptions.DataError(u"allow_comments set to False, but comments are there") return None - def publishItem(self, client, mb_data, service, node): + def publishItem(self, client, mb_data, service, node, session): log.debug(u"uploading item [{id}]: {title}".format(id=mb_data['id'], title=mb_data.get('title',''))) return self._m.send(client, mb_data, service, node) @defer.inlineCallbacks - def itemFilters(self, client, mb_data, options): + def itemFilters(self, client, mb_data, session, options): """Apply filters according to options modify mb_data in place diff -r cdaa58e14553 -r 2c2b826b0bb3 src/plugins/plugin_import.py --- a/src/plugins/plugin_import.py Sun Oct 01 12:21:23 2017 +0200 +++ b/src/plugins/plugin_import.py Fri Oct 06 08:52:51 2017 +0200 @@ -65,14 +65,14 @@ import_handler.register = partial(self.register, import_handler) import_handler.unregister = partial(self.unregister, import_handler) import_handler.importers = {} - def _import(name, location, options, pubsub_service, profile): - return self._doImport(import_handler, name, location, options, pubsub_service, profile) + def _import(name, location, options, pubsub_service, pubsub_node, profile): + return self._doImport(import_handler, name, location, options, pubsub_service, pubsub_node, profile) def _importList(): return self.listImporters(import_handler) def _importDesc(name): return self.getDescription(import_handler, name) - self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}ss', out_sign='s', method=_import, async=True) + self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}sss', out_sign='s', method=_import, async=True) self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList) self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc) @@ -100,7 +100,7 @@ else: return importer.short_desc, importer.long_desc - def _doImport(self, import_handler, name, location, options, pubsub_service='', profile=C.PROF_KEY_NONE): + def _doImport(self, import_handler, name, location, options, pubsub_service='', pubsub_node='', profile=C.PROF_KEY_NONE): client = self.host.getClient(profile) options = {key: unicode(value) for key, value in options.iteritems()} for option in import_handler.BOOL_OPTIONS: @@ -108,10 +108,10 @@ options[option] = C.bool(options[option]) except KeyError: pass - return self.doImport(client, import_handler, unicode(name), unicode(location), options) + return self.doImport(client, import_handler, unicode(name), unicode(location), options, pubsub_service or None, pubsub_node or None) @defer.inlineCallbacks - def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None): + def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None, pubsub_node=None): """Import data @param import_handler(object): instance of the import handler @@ -122,6 +122,8 @@ @param options(dict, None): extra options. @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported None to use profile's server + @param pubsub_node(unicode, None): PubSub node to use + None to use importer's default node @return (unicode): progress id """ if options is None: @@ -157,17 +159,19 @@ } self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile) self.host.bridge.progressStarted(progress_id, metadata, client.profile) - url_redirect = {} - self.recursiveImport(client, import_handler, items_import_data, progress_id, options, url_redirect) + session = {} # session data, can be use by importers + self.recursiveImport(client, import_handler, items_import_data, progress_id, session, options, None, pubsub_service, pubsub_node) defer.returnValue(progress_id) @defer.inlineCallbacks - def recursiveImport(self, client, import_handler, items_import_data, progress_id, options, return_data=None, service=None, node=None, depth=0): + def recursiveImport(self, client, import_handler, items_import_data, progress_id, session, options, return_data=None, service=None, node=None, depth=0): """Do the import recursively @param import_handler(object): instance of the import handler @param items_import_data(iterable): iterable of data as specified in [register] @param progress_id(unicode): id of progression + @param session(dict): data for this import session + can be used by importer so store any useful data @param options(dict): import options @param return_data(dict): data to return on progressFinished @param service(jid.JID, None): PubSub service to use @@ -177,15 +181,16 @@ if return_data is None: return_data = {} for idx, item_import_data in enumerate(items_import_data): - item_data = yield import_handler.importItem(client, item_import_data, options, return_data, service, node) - yield import_handler.itemFilters(client, item_data, options) - recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, options) - yield import_handler.publishItem(client, item_data, service, node) + item_data = yield import_handler.importItem(client, item_import_data, session, options, return_data, service, node) + yield import_handler.itemFilters(client, item_data, session, options) + recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, session, options) + yield import_handler.publishItem(client, item_data, service, node, session) if recurse_kwargs is not None: recurse_kwargs['client'] = client recurse_kwargs['import_handler'] = import_handler recurse_kwargs['progress_id'] = progress_id + recurse_kwargs['session'] = session recurse_kwargs.setdefault('options', options) recurse_kwargs['return_data'] = return_data recurse_kwargs['depth'] = depth + 1