libervia-backend: sat/plugins/plugin

annotate sat/plugins/plugin_import.py @ 3040:fee60f17ebac

jp: jp asyncio port: /!\ this commit is huge. Jp is temporarily not working with `dbus` bridge /!\ This patch implements the port of jp to asyncio, so it is now correctly using the bridge asynchronously, and it can be used with bridges like `pb`. This also simplify the code, notably for things which were previously implemented with many callbacks (like pagination with RSM). During the process, some behaviours have been modified/fixed, in jp and backends, check diff for details.

author	Goffi <goffi@goffi.org>
date	Wed, 25 Sep 2019 08:56:41 +0200
parents	ab2696e34d29
children	9d0df638c8b4

rev	line source
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	1 #!/usr/bin/env python3
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	2 # -- coding: utf-8 --
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	3
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	4 # SàT plugin for generic data import handling
2771 003b8b4b56a7 date update Goffi <goffi@goffi.org> parents: 2624 diff changeset	5 # Copyright (C) 2009-2019 Jérôme Poisson (goffi@goffi.org)
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	6
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	7 # This program is free software: you can redistribute it and/or modify
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	8 # it under the terms of the GNU Affero General Public License as published by
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	9 # the Free Software Foundation, either version 3 of the License, or
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	10 # (at your option) any later version.
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	11
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	12 # This program is distributed in the hope that it will be useful,
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	15 # GNU Affero General Public License for more details.
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	16
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	17 # You should have received a copy of the GNU Affero General Public License
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	19
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	20 from sat.core.i18n import _
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	21 from sat.core.constants import Const as C
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	22 from sat.core.log import getLogger
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	23
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	24 log = getLogger(__name__)
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	25 from twisted.internet import defer
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	26 from sat.core import exceptions
2436 b52e78cc86ed plugin import: fixed deserialisation of pubsub service Goffi <goffi@goffi.org> parents: 2414 diff changeset	27 from twisted.words.protocols.jabber import jid
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	28 from functools import partial
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	29 import collections
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	30 import uuid
2396 66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping: Goffi <goffi@goffi.org> parents: 2390 diff changeset	31 import json
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	32
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	33
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	34 PLUGIN_INFO = {
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	35 C.PI_NAME: "import",
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	36 C.PI_IMPORT_NAME: "IMPORT",
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	37 C.PI_TYPE: C.PLUG_TYPE_IMPORT,
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	38 C.PI_DEPENDENCIES: [],
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	39 C.PI_MAIN: "ImportPlugin",
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	40 C.PI_HANDLER: "no",
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	41 C.PI_DESCRIPTION: _("""Generic import plugin, base for specialized importers"""),
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	42 }
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	43
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	44 Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc"))
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	45
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	46
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	47 class ImportPlugin(object):
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	48 def __init__(self, host):
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	49 log.info(_("plugin Import initialization"))
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	50 self.host = host
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	51
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	52 def initialize(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	53 """Initialize a specialized import handler
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	54
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	55 @param import_handler(object): specialized import handler instance
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	56 must have the following methods:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	57 - importItem: import a single main item (i.e. prepare data for publishing)
2390 f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling: Goffi <goffi@goffi.org> parents: 2370 diff changeset	58 - importSubitems: import sub items (i.e. items linked to main item, e.g. comments).
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling: Goffi <goffi@goffi.org> parents: 2370 diff changeset	59 Must return a dict with kwargs for recursiveImport if items are to be imported recursively.
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling: Goffi <goffi@goffi.org> parents: 2370 diff changeset	60 At least "items_import_data", "service" and "node" keys must be provided.
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling: Goffi <goffi@goffi.org> parents: 2370 diff changeset	61 if None is returned, no recursion will be done to import subitems, but import can still be done directly by the method.
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	62 - publishItem: actualy publish an item
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	63 - itemFilters: modify item according to options
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	64 @param name(unicode): import handler name
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	65 """
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	66 assert name == name.lower().strip()
3040 fee60f17ebac jp: jp asyncio port: Goffi <goffi@goffi.org> parents: 3028 diff changeset	67 log.info(_(f"initializing {name} import handler"))
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	68 import_handler.name = name
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	69 import_handler.register = partial(self.register, import_handler)
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	70 import_handler.unregister = partial(self.unregister, import_handler)
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	71 import_handler.importers = {}
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	72
2370 2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data: Goffi <goffi@goffi.org> parents: 2369 diff changeset	73 def _import(name, location, options, pubsub_service, pubsub_node, profile):
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	74 return self._doImport(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	75 import_handler,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	76 name,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	77 location,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	78 options,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	79 pubsub_service,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	80 pubsub_node,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	81 profile,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	82 )
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	83
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	84 def _importList():
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	85 return self.listImporters(import_handler)
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	86
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	87 def _importDesc(name):
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	88 return self.getDescription(import_handler, name)
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	89
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	90 self.host.bridge.addMethod(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	91 name + "Import",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	92 ".plugin",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	93 in_sign="ssa{ss}sss",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	94 out_sign="s",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	95 method=_import,
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	96 async_=True,
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	97 )
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	98 self.host.bridge.addMethod(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	99 name + "ImportList",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	100 ".plugin",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	101 in_sign="",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	102 out_sign="a(ss)",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	103 method=_importList,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	104 )
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	105 self.host.bridge.addMethod(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	106 name + "ImportDesc",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	107 ".plugin",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	108 in_sign="s",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	109 out_sign="(ss)",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	110 method=_importDesc,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	111 )
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	112
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	113 def getProgress(self, import_handler, progress_id, profile):
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	114 client = self.host.getClient(profile)
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	115 return client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	116
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	117 def listImporters(self, import_handler):
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	118 importers = list(import_handler.importers.keys())
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	119 importers.sort()
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	120 return [
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	121 (name, import_handler.importers[name].short_desc)
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	122 for name in import_handler.importers
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	123 ]
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	124
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	125 def getDescription(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	126 """Return import short and long descriptions
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	127
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	128 @param name(unicode): importer name
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	129 @return (tuple[unicode,unicode]): short and long description
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	130 """
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	131 try:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	132 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	133 except KeyError:
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	134 raise exceptions.NotFound(
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	135 "{handler_name} importer not found [{name}]".format(
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	136 handler_name=import_handler.name, name=name
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	137 )
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	138 )
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	139 else:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	140 return importer.short_desc, importer.long_desc
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	141
3040 fee60f17ebac jp: jp asyncio port: Goffi <goffi@goffi.org> parents: 3028 diff changeset	142 def _doImport(self, import_handler, name, location, options, pubsub_service="",
fee60f17ebac jp: jp asyncio port: Goffi <goffi@goffi.org> parents: 3028 diff changeset	143 pubsub_node="", profile=C.PROF_KEY_NONE):
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	144 client = self.host.getClient(profile)
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	145 options = {key: str(value) for key, value in options.items()}
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	146 for option in import_handler.BOOL_OPTIONS:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	147 try:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	148 options[option] = C.bool(options[option])
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	149 except KeyError:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	150 pass
2396 66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping: Goffi <goffi@goffi.org> parents: 2390 diff changeset	151 for option in import_handler.JSON_OPTIONS:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping: Goffi <goffi@goffi.org> parents: 2390 diff changeset	152 try:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping: Goffi <goffi@goffi.org> parents: 2390 diff changeset	153 options[option] = json.loads(options[option])
3040 fee60f17ebac jp: jp asyncio port: Goffi <goffi@goffi.org> parents: 3028 diff changeset	154 except KeyError:
fee60f17ebac jp: jp asyncio port: Goffi <goffi@goffi.org> parents: 3028 diff changeset	155 pass
2396 66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping: Goffi <goffi@goffi.org> parents: 2390 diff changeset	156 except ValueError:
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	157 raise exceptions.DataError(
3040 fee60f17ebac jp: jp asyncio port: Goffi <goffi@goffi.org> parents: 3028 diff changeset	158 _(f"invalid json option: {option}")
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	159 )
2436 b52e78cc86ed plugin import: fixed deserialisation of pubsub service Goffi <goffi@goffi.org> parents: 2414 diff changeset	160 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	161 return self.doImport(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	162 client,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	163 import_handler,
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	164 str(name),
ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	165 str(location),
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	166 options,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	167 pubsub_service,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	168 pubsub_node or None,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	169 )
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	170
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	171 @defer.inlineCallbacks
3040 fee60f17ebac jp: jp asyncio port: Goffi <goffi@goffi.org> parents: 3028 diff changeset	172 def doImport(self, client, import_handler, name, location, options=None,
fee60f17ebac jp: jp asyncio port: Goffi <goffi@goffi.org> parents: 3028 diff changeset	173 pubsub_service=None, pubsub_node=None,):
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	174 """Import data
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	175
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	176 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	177 @param name(unicode): name of the importer
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	178 @param location(unicode): location of the data to import
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	179 can be an url, a file path, or anything which make sense
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	180 check importer description for more details
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	181 @param options(dict, None): extra options.
3040 fee60f17ebac jp: jp asyncio port: Goffi <goffi@goffi.org> parents: 3028 diff changeset	182 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be
fee60f17ebac jp: jp asyncio port: Goffi <goffi@goffi.org> parents: 3028 diff changeset	183 imported.
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	184 None to use profile's server
2370 2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data: Goffi <goffi@goffi.org> parents: 2369 diff changeset	185 @param pubsub_node(unicode, None): PubSub node to use
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data: Goffi <goffi@goffi.org> parents: 2369 diff changeset	186 None to use importer's default node
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	187 @return (unicode): progress id
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	188 """
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	189 if options is None:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	190 options = {}
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	191 else:
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	192 for opt_name, opt_default in import_handler.OPT_DEFAULTS.items():
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	193 # we want a filled options dict, with all empty or False values removed
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	194 try:
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	195 value = options[opt_name]
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	196 except KeyError:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	197 if opt_default:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	198 options[opt_name] = opt_default
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	199 else:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	200 if not value:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	201 del options[opt_name]
2390 f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling: Goffi <goffi@goffi.org> parents: 2370 diff changeset	202
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	203 try:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	204 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	205 except KeyError:
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	206 raise exceptions.NotFound("Importer [{}] not found".format(name))
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	207 items_import_data, items_count = yield importer.callback(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	208 client, location, options
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	209 )
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	210 progress_id = str(uuid.uuid4())
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	211 try:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	212 _import = client._import
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	213 except AttributeError:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	214 _import = client._import = {}
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	215 progress_data = _import.setdefault(import_handler.name, {})
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	216 progress_data[progress_id] = {"position": "0"}
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	217 if items_count is not None:
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	218 progress_data[progress_id]["size"] = str(items_count)
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	219 metadata = {
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	220 "name": "{}: {}".format(name, location),
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	221 "direction": "out",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	222 "type": import_handler.name.upper() + "_IMPORT",
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	223 }
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	224 self.host.registerProgressCb(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	225 progress_id,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	226 partial(self.getProgress, import_handler),
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	227 metadata,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	228 profile=client.profile,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	229 )
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	230 self.host.bridge.progressStarted(progress_id, metadata, client.profile)
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	231 session = { # session data, can be used by importers
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	232 "root_service": pubsub_service,
ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	233 "root_node": pubsub_node,
2390 f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling: Goffi <goffi@goffi.org> parents: 2370 diff changeset	234 }
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	235 self.recursiveImport(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	236 client,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	237 import_handler,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	238 items_import_data,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	239 progress_id,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	240 session,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	241 options,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	242 None,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	243 pubsub_service,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	244 pubsub_node,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	245 )
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	246 defer.returnValue(progress_id)
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	247
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	248 @defer.inlineCallbacks
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	249 def recursiveImport(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	250 self,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	251 client,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	252 import_handler,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	253 items_import_data,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	254 progress_id,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	255 session,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	256 options,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	257 return_data=None,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	258 service=None,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	259 node=None,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	260 depth=0,
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	261 ):
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	262 """Do the import recursively
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	263
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	264 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	265 @param items_import_data(iterable): iterable of data as specified in [register]
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	266 @param progress_id(unicode): id of progression
2370 2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data: Goffi <goffi@goffi.org> parents: 2369 diff changeset	267 @param session(dict): data for this import session
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data: Goffi <goffi@goffi.org> parents: 2369 diff changeset	268 can be used by importer so store any useful data
2390 f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling: Goffi <goffi@goffi.org> parents: 2370 diff changeset	269 "root_service" and "root_node" are set to the main pubsub service and node of the import
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	270 @param options(dict): import options
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	271 @param return_data(dict): data to return on progressFinished
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	272 @param service(jid.JID, None): PubSub service to use
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	273 @param node(unicode, None): PubSub node to use
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	274 @param depth(int): level of recursion
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	275 """
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	276 if return_data is None:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	277 return_data = {}
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	278 for idx, item_import_data in enumerate(items_import_data):
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	279 item_data = yield import_handler.importItem(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	280 client, item_import_data, session, options, return_data, service, node
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	281 )
2370 2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data: Goffi <goffi@goffi.org> parents: 2369 diff changeset	282 yield import_handler.itemFilters(client, item_data, session, options)
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	283 recurse_kwargs = yield import_handler.importSubItems(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	284 client, item_import_data, item_data, session, options
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	285 )
2370 2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data: Goffi <goffi@goffi.org> parents: 2369 diff changeset	286 yield import_handler.publishItem(client, item_data, service, node, session)
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	287
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	288 if recurse_kwargs is not None:
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	289 recurse_kwargs["client"] = client
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	290 recurse_kwargs["import_handler"] = import_handler
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	291 recurse_kwargs["progress_id"] = progress_id
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	292 recurse_kwargs["session"] = session
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	293 recurse_kwargs.setdefault("options", options)
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	294 recurse_kwargs["return_data"] = return_data
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	295 recurse_kwargs["depth"] = depth + 1
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	296 log.debug(_("uploading subitems"))
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	297 yield self.recursiveImport(**recurse_kwargs)
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	298
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	299 if depth == 0:
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	300 client._import[import_handler.name][progress_id]["position"] = str(
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	301 idx + 1
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	302 )
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	303
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	304 if depth == 0:
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	305 self.host.bridge.progressFinished(progress_id, return_data, client.profile)
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	306 self.host.removeProgressCb(progress_id, client.profile)
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	307 del client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	308
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	309 def register(self, import_handler, name, callback, short_desc="", long_desc=""):
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	310 """Register an Importer method
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	311
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	312 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	313 @param callback(callable): method to call:
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	314 the signature must be (client, location, options) (cf. [doImport])
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	315 the importer must return a tuple with (items_import_data, items_count)
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	316 items_import_data(iterable[dict]) data specific to specialized importer
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	317 cf. importItem docstring of specialized importer for details
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	318 items_count (int, None) indicate the total number of items (without subitems)
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	319 useful to display a progress indicator when the iterator is a generator
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	320 use None if you can't guess the total number of items
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	321 @param short_desc(unicode): one line description of the importer
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	322 @param long_desc(unicode): long description of the importer, its options, etc.
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	323 """
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	324 name = name.lower()
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	325 if name in import_handler.importers:
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	326 raise exceptions.ConflictError(
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	327 _(
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	328 "An {handler_name} importer with the name {name} already exist"
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	329 ).format(handler_name=import_handler.name, name=name)
56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	330 )
2369 cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	331 import_handler.importers[name] = Importer(callback, short_desc, long_desc)
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	332
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	333 def unregister(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin: Goffi <goffi@goffi.org> parents: diff changeset	334 del import_handler.importers[name]

Mercurial > libervia-backend

annotate sat/plugins/plugin_import.py @ 3040:fee60f17ebac