Mercurial > libervia-backend

annotate sat/plugins/plugin_import.py @ 2862:b2e898384c25

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
quick_frontend (app): progress callbacks handling: in addition to listeners, a frontend can now register progression callback and errback which are called when the given progressing action is finished. Those callbacks can be registered using registerProgressCbs or specified in actionManager.
author Goffi <goffi@goffi.org>
date Wed, 20 Mar 2019 09:08:47 +0100
parents 003b8b4b56a7
children ab2696e34d29
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
1 #!/usr/bin/env python2
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
2 # -*- coding: utf-8 -*-
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
3
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
4 # SàT plugin for generic data import handling
2771
003b8b4b56a7 date update
Goffi <goffi@goffi.org>
parents: 2624
diff changeset
5 # Copyright (C) 2009-2019 Jérôme Poisson (goffi@goffi.org)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
6
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
7 # This program is free software: you can redistribute it and/or modify
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
8 # it under the terms of the GNU Affero General Public License as published by
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
9 # the Free Software Foundation, either version 3 of the License, or
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
10 # (at your option) any later version.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
11
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
12 # This program is distributed in the hope that it will be useful,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
15 # GNU Affero General Public License for more details.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
16
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
17 # You should have received a copy of the GNU Affero General Public License
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
19
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
20 from sat.core.i18n import _
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
21 from sat.core.constants import Const as C
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
22 from sat.core.log import getLogger
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
23
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
24 log = getLogger(__name__)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
25 from twisted.internet import defer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
26 from sat.core import exceptions
2436
b52e78cc86ed plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents: 2414
diff changeset
27 from twisted.words.protocols.jabber import jid
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
28 from functools import partial
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
29 import collections
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
30 import uuid
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
31 import json
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
32
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
33
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
34 PLUGIN_INFO = {
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
35 C.PI_NAME: "import",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
36 C.PI_IMPORT_NAME: "IMPORT",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
37 C.PI_TYPE: C.PLUG_TYPE_IMPORT,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
38 C.PI_DEPENDENCIES: [],
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
39 C.PI_MAIN: "ImportPlugin",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
40 C.PI_HANDLER: "no",
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
41 C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers"""),
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
42 }
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
43
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
44 Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc"))
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
45
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
46
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
47 class ImportPlugin(object):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
48 def __init__(self, host):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
49 log.info(_("plugin Import initialization"))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
50 self.host = host
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
51
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
52 def initialize(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
53 """Initialize a specialized import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
54
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
55 @param import_handler(object): specialized import handler instance
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
56 must have the following methods:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
57 - importItem: import a single main item (i.e. prepare data for publishing)
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
58 - importSubitems: import sub items (i.e. items linked to main item, e.g. comments).
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
59 Must return a dict with kwargs for recursiveImport if items are to be imported recursively.
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
60 At least "items_import_data", "service" and "node" keys must be provided.
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
61 if None is returned, no recursion will be done to import subitems, but import can still be done directly by the method.
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
62 - publishItem: actualy publish an item
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
63 - itemFilters: modify item according to options
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
64 @param name(unicode): import handler name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
65 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
66 assert name == name.lower().strip()
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
67 log.info(_(u"initializing {name} import handler").format(name=name))
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
68 import_handler.name = name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
69 import_handler.register = partial(self.register, import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
70 import_handler.unregister = partial(self.unregister, import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
71 import_handler.importers = {}
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
72
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
73 def _import(name, location, options, pubsub_service, pubsub_node, profile):
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
74 return self._doImport(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
75 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
76 name,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
77 location,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
78 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
79 pubsub_service,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
80 pubsub_node,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
81 profile,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
82 )
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
83
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
84 def _importList():
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
85 return self.listImporters(import_handler)
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
86
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
87 def _importDesc(name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
88 return self.getDescription(import_handler, name)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
89
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
90 self.host.bridge.addMethod(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
91 name + "Import",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
92 ".plugin",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
93 in_sign="ssa{ss}sss",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
94 out_sign="s",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
95 method=_import,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
96 async=True,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
97 )
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
98 self.host.bridge.addMethod(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
99 name + "ImportList",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
100 ".plugin",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
101 in_sign="",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
102 out_sign="a(ss)",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
103 method=_importList,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
104 )
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
105 self.host.bridge.addMethod(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
106 name + "ImportDesc",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
107 ".plugin",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
108 in_sign="s",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
109 out_sign="(ss)",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
110 method=_importDesc,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
111 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
112
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
113 def getProgress(self, import_handler, progress_id, profile):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
114 client = self.host.getClient(profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
115 return client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
116
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
117 def listImporters(self, import_handler):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
118 importers = import_handler.importers.keys()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
119 importers.sort()
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
120 return [
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
121 (name, import_handler.importers[name].short_desc)
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
122 for name in import_handler.importers
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
123 ]
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
124
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
125 def getDescription(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
126 """Return import short and long descriptions
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
127
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
128 @param name(unicode): importer name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
129 @return (tuple[unicode,unicode]): short and long description
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
130 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
131 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
132 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
133 except KeyError:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
134 raise exceptions.NotFound(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
135 u"{handler_name} importer not found [{name}]".format(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
136 handler_name=import_handler.name, name=name
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
137 )
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
138 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
139 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
140 return importer.short_desc, importer.long_desc
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
141
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
142 def _doImport(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
143 self,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
144 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
145 name,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
146 location,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
147 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
148 pubsub_service="",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
149 pubsub_node="",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
150 profile=C.PROF_KEY_NONE,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
151 ):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
152 client = self.host.getClient(profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
153 options = {key: unicode(value) for key, value in options.iteritems()}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
154 for option in import_handler.BOOL_OPTIONS:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
155 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
156 options[option] = C.bool(options[option])
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
157 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
158 pass
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
159 for option in import_handler.JSON_OPTIONS:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
160 try:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
161 options[option] = json.loads(options[option])
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
162 except ValueError:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
163 raise exceptions.DataError(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
164 _(u"invalid json option: {name}").format(name=option)
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
165 )
2436
b52e78cc86ed plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents: 2414
diff changeset
166 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
167 return self.doImport(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
168 client,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
169 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
170 unicode(name),
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
171 unicode(location),
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
172 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
173 pubsub_service,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
174 pubsub_node or None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
175 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
176
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
177 @defer.inlineCallbacks
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
178 def doImport(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
179 self,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
180 client,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
181 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
182 name,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
183 location,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
184 options=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
185 pubsub_service=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
186 pubsub_node=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
187 ):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
188 """Import data
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
189
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
190 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
191 @param name(unicode): name of the importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
192 @param location(unicode): location of the data to import
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
193 can be an url, a file path, or anything which make sense
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
194 check importer description for more details
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
195 @param options(dict, None): extra options.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
196 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
197 None to use profile's server
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
198 @param pubsub_node(unicode, None): PubSub node to use
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
199 None to use importer's default node
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
200 @return (unicode): progress id
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
201 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
202 if options is None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
203 options = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
204 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
205 for opt_name, opt_default in import_handler.OPT_DEFAULTS.iteritems():
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
206 # we want a filled options dict, with all empty or False values removed
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
207 try:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
208 value = options[opt_name]
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
209 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
210 if opt_default:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
211 options[opt_name] = opt_default
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
212 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
213 if not value:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
214 del options[opt_name]
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
215
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
216 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
217 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
218 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
219 raise exceptions.NotFound(u"Importer [{}] not found".format(name))
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
220 items_import_data, items_count = yield importer.callback(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
221 client, location, options
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
222 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
223 progress_id = unicode(uuid.uuid4())
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
224 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
225 _import = client._import
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
226 except AttributeError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
227 _import = client._import = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
228 progress_data = _import.setdefault(import_handler.name, {})
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
229 progress_data[progress_id] = {u"position": "0"}
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
230 if items_count is not None:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
231 progress_data[progress_id]["size"] = unicode(items_count)
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
232 metadata = {
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
233 "name": u"{}: {}".format(name, location),
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
234 "direction": "out",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
235 "type": import_handler.name.upper() + "_IMPORT",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
236 }
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
237 self.host.registerProgressCb(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
238 progress_id,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
239 partial(self.getProgress, import_handler),
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
240 metadata,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
241 profile=client.profile,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
242 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
243 self.host.bridge.progressStarted(progress_id, metadata, client.profile)
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
244 session = { #  session data, can be used by importers
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
245 u"root_service": pubsub_service,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
246 u"root_node": pubsub_node,
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
247 }
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
248 self.recursiveImport(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
249 client,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
250 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
251 items_import_data,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
252 progress_id,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
253 session,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
254 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
255 None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
256 pubsub_service,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
257 pubsub_node,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
258 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
259 defer.returnValue(progress_id)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
260
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
261 @defer.inlineCallbacks
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
262 def recursiveImport(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
263 self,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
264 client,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
265 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
266 items_import_data,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
267 progress_id,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
268 session,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
269 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
270 return_data=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
271 service=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
272 node=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
273 depth=0,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
274 ):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
275 """Do the import recursively
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
276
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
277 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
278 @param items_import_data(iterable): iterable of data as specified in [register]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
279 @param progress_id(unicode): id of progression
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
280 @param session(dict): data for this import session
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
281 can be used by importer so store any useful data
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
282 "root_service" and "root_node" are set to the main pubsub service and node of the import
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
283 @param options(dict): import options
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
284 @param return_data(dict): data to return on progressFinished
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
285 @param service(jid.JID, None): PubSub service to use
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
286 @param node(unicode, None): PubSub node to use
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
287 @param depth(int): level of recursion
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
288 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
289 if return_data is None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
290 return_data = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
291 for idx, item_import_data in enumerate(items_import_data):
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
292 item_data = yield import_handler.importItem(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
293 client, item_import_data, session, options, return_data, service, node
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
294 )
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
295 yield import_handler.itemFilters(client, item_data, session, options)
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
296 recurse_kwargs = yield import_handler.importSubItems(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
297 client, item_import_data, item_data, session, options
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
298 )
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
299 yield import_handler.publishItem(client, item_data, service, node, session)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
300
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
301 if recurse_kwargs is not None:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
302 recurse_kwargs["client"] = client
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
303 recurse_kwargs["import_handler"] = import_handler
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
304 recurse_kwargs["progress_id"] = progress_id
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
305 recurse_kwargs["session"] = session
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
306 recurse_kwargs.setdefault("options", options)
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
307 recurse_kwargs["return_data"] = return_data
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
308 recurse_kwargs["depth"] = depth + 1
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
309 log.debug(_(u"uploading subitems"))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
310 yield self.recursiveImport(**recurse_kwargs)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
311
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
312 if depth == 0:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
313 client._import[import_handler.name][progress_id]["position"] = unicode(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
314 idx + 1
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
315 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
316
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
317 if depth == 0:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
318 self.host.bridge.progressFinished(progress_id, return_data, client.profile)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
319 self.host.removeProgressCb(progress_id, client.profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
320 del client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
321
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
322 def register(self, import_handler, name, callback, short_desc="", long_desc=""):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
323 """Register an Importer method
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
324
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
325 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
326 @param callback(callable): method to call:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
327 the signature must be (client, location, options) (cf. [doImport])
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
328 the importer must return a tuple with (items_import_data, items_count)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
329 items_import_data(iterable[dict]) data specific to specialized importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
330 cf. importItem docstring of specialized importer for details
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
331 items_count (int, None) indicate the total number of items (without subitems)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
332 useful to display a progress indicator when the iterator is a generator
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
333 use None if you can't guess the total number of items
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
334 @param short_desc(unicode): one line description of the importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
335 @param long_desc(unicode): long description of the importer, its options, etc.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
336 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
337 name = name.lower()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
338 if name in import_handler.importers:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
339 raise exceptions.ConflictError(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
340 _(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
341 u"An {handler_name} importer with the name {name} already exist"
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
342 ).format(handler_name=import_handler.name, name=name)
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
343 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
344 import_handler.importers[name] = Importer(callback, short_desc, long_desc)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
345
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
346 def unregister(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
347 del import_handler.importers[name]