Mercurial > libervia-backend

annotate libervia/backend/plugins/plugin_import.py @ 4340:ea72364131d5 default tip @

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
doc (components): Update Email Gateway documentation: A section has been added to explain how attachments are handled. fix 453
author Goffi <goffi@goffi.org>
date Tue, 03 Dec 2024 00:53:18 +0100
parents 0d7bb4df2343
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
1 #!/usr/bin/env python3
3137
559a625a236b fixed shebangs
Goffi <goffi@goffi.org>
parents: 3136
diff changeset
2
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
3
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
4 # SàT plugin for generic data import handling
3479
be6d91572633 date update
Goffi <goffi@goffi.org>
parents: 3137
diff changeset
5 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
6
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
7 # This program is free software: you can redistribute it and/or modify
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
8 # it under the terms of the GNU Affero General Public License as published by
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
9 # the Free Software Foundation, either version 3 of the License, or
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
10 # (at your option) any later version.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
11
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
12 # This program is distributed in the hope that it will be useful,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
15 # GNU Affero General Public License for more details.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
16
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
17 # You should have received a copy of the GNU Affero General Public License
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
19
4071
4b842c1fb686 refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents: 4037
diff changeset
20 from libervia.backend.core.i18n import _
4b842c1fb686 refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents: 4037
diff changeset
21 from libervia.backend.core.constants import Const as C
4b842c1fb686 refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents: 4037
diff changeset
22 from libervia.backend.core.log import getLogger
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
23
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
24 log = getLogger(__name__)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
25 from twisted.internet import defer
4071
4b842c1fb686 refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents: 4037
diff changeset
26 from libervia.backend.core import exceptions
2436
b52e78cc86ed plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents: 2414
diff changeset
27 from twisted.words.protocols.jabber import jid
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
28 from functools import partial
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
29 import collections
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
30 import uuid
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
31 import json
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
32
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
33
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
34 PLUGIN_INFO = {
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
35 C.PI_NAME: "import",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
36 C.PI_IMPORT_NAME: "IMPORT",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
37 C.PI_TYPE: C.PLUG_TYPE_IMPORT,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
38 C.PI_DEPENDENCIES: [],
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
39 C.PI_MAIN: "ImportPlugin",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
40 C.PI_HANDLER: "no",
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
41 C.PI_DESCRIPTION: _("""Generic import plugin, base for specialized importers"""),
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
42 }
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
43
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
44 Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc"))
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
45
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
46
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
47 class ImportPlugin(object):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
48 def __init__(self, host):
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
49 log.info(_("plugin import initialization"))
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
50 self.host = host
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
51
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
52 def initialize(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
53 """Initialize a specialized import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
54
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
55 @param import_handler(object): specialized import handler instance
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
56 must have the following methods:
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
57 - import_item: import a single main item (i.e. prepare data for publishing)
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
58 - importSubitems: import sub items (i.e. items linked to main item, e.g. comments).
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
59 Must return a dict with kwargs for recursive_import if items are to be imported recursively.
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
60 At least "items_import_data", "service" and "node" keys must be provided.
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
61 if None is returned, no recursion will be done to import subitems, but import can still be done directly by the method.
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
62 - publish_item: actualy publish an item
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
63 - item_filters: modify item according to options
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
64 @param name(unicode): import handler name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
65 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
66 assert name == name.lower().strip()
3568
04283582966f core, frontends: fix invalid translatable strings.
Goffi <goffi@goffi.org>
parents: 3479
diff changeset
67 log.info(_("initializing {name} import handler").format(name=name))
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
68 import_handler.name = name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
69 import_handler.register = partial(self.register, import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
70 import_handler.unregister = partial(self.unregister, import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
71 import_handler.importers = {}
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
72
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
73 def _import(name, location, options, pubsub_service, pubsub_node, profile):
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
74 return self._do_import(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
75 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
76 name,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
77 location,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
78 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
79 pubsub_service,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
80 pubsub_node,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
81 profile,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
82 )
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
83
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
84 def _import_list():
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
85 return self.list_importers(import_handler)
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
86
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
87 def _import_desc(name):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
88 return self.getDescription(import_handler, name)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
89
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
90 self.host.bridge.add_method(
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
91 name + "import",
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
92 ".plugin",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
93 in_sign="ssa{ss}sss",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
94 out_sign="s",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
95 method=_import,
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
96 async_=True,
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
97 )
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
98 self.host.bridge.add_method(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
99 name + "ImportList",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
100 ".plugin",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
101 in_sign="",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
102 out_sign="a(ss)",
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
103 method=_import_list,
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
104 )
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
105 self.host.bridge.add_method(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
106 name + "ImportDesc",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
107 ".plugin",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
108 in_sign="s",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
109 out_sign="(ss)",
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
110 method=_import_desc,
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
111 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
112
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
113 def get_progress(self, import_handler, progress_id, profile):
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
114 client = self.host.get_client(profile)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
115 return client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
116
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
117 def list_importers(self, import_handler):
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
118 importers = list(import_handler.importers.keys())
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
119 importers.sort()
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
120 return [
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
121 (name, import_handler.importers[name].short_desc)
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
122 for name in import_handler.importers
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
123 ]
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
124
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
125 def getDescription(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
126 """Return import short and long descriptions
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
127
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
128 @param name(unicode): importer name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
129 @return (tuple[unicode,unicode]): short and long description
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
130 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
131 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
132 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
133 except KeyError:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
134 raise exceptions.NotFound(
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
135 "{handler_name} importer not found [{name}]".format(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
136 handler_name=import_handler.name, name=name
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
137 )
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
138 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
139 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
140 return importer.short_desc, importer.long_desc
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
141
4270
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
142 def _do_import(
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
143 self,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
144 import_handler,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
145 name,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
146 location,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
147 options,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
148 pubsub_service="",
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
149 pubsub_node="",
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
150 profile=C.PROF_KEY_NONE,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
151 ):
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
152 client = self.host.get_client(profile)
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
153 options = {key: str(value) for key, value in options.items()}
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
154 for option in import_handler.BOOL_OPTIONS:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
155 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
156 options[option] = C.bool(options[option])
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
157 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
158 pass
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
159 for option in import_handler.JSON_OPTIONS:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
160 try:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
161 options[option] = json.loads(options[option])
3040
fee60f17ebac jp: jp asyncio port:
Goffi <goffi@goffi.org>
parents: 3028
diff changeset
162 except KeyError:
fee60f17ebac jp: jp asyncio port:
Goffi <goffi@goffi.org>
parents: 3028
diff changeset
163 pass
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
164 except ValueError:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
165 raise exceptions.DataError(
3568
04283582966f core, frontends: fix invalid translatable strings.
Goffi <goffi@goffi.org>
parents: 3479
diff changeset
166 _("invalid json option: {option}").format(option=option)
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
167 )
2436
b52e78cc86ed plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents: 2414
diff changeset
168 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
169 return self.do_import(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
170 client,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
171 import_handler,
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
172 str(name),
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
173 str(location),
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
174 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
175 pubsub_service,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
176 pubsub_node or None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
177 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
178
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
179 @defer.inlineCallbacks
4270
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
180 def do_import(
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
181 self,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
182 client,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
183 import_handler,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
184 name,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
185 location,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
186 options=None,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
187 pubsub_service=None,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
188 pubsub_node=None,
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
189 ):
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
190 """import data
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
191
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
192 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
193 @param name(unicode): name of the importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
194 @param location(unicode): location of the data to import
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
195 can be an url, a file path, or anything which make sense
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
196 check importer description for more details
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
197 @param options(dict, None): extra options.
3040
fee60f17ebac jp: jp asyncio port:
Goffi <goffi@goffi.org>
parents: 3028
diff changeset
198 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be
fee60f17ebac jp: jp asyncio port:
Goffi <goffi@goffi.org>
parents: 3028
diff changeset
199 imported.
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
200 None to use profile's server
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
201 @param pubsub_node(unicode, None): PubSub node to use
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
202 None to use importer's default node
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
203 @return (unicode): progress id
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
204 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
205 if options is None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
206 options = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
207 else:
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
208 for opt_name, opt_default in import_handler.OPT_DEFAULTS.items():
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
209 # we want a filled options dict, with all empty or False values removed
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
210 try:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
211 value = options[opt_name]
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
212 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
213 if opt_default:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
214 options[opt_name] = opt_default
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
215 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
216 if not value:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
217 del options[opt_name]
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
218
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
219 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
220 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
221 except KeyError:
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
222 raise exceptions.NotFound("Importer [{}] not found".format(name))
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
223 items_import_data, items_count = yield importer.callback(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
224 client, location, options
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
225 )
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
226 progress_id = str(uuid.uuid4())
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
227 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
228 _import = client._import
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
229 except AttributeError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
230 _import = client._import = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
231 progress_data = _import.setdefault(import_handler.name, {})
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
232 progress_data[progress_id] = {"position": "0"}
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
233 if items_count is not None:
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
234 progress_data[progress_id]["size"] = str(items_count)
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
235 metadata = {
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
236 "name": "{}: {}".format(name, location),
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
237 "direction": "out",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
238 "type": import_handler.name.upper() + "_IMPORT",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
239 }
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
240 self.host.register_progress_cb(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
241 progress_id,
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
242 partial(self.get_progress, import_handler),
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
243 metadata,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
244 profile=client.profile,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
245 )
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
246 self.host.bridge.progress_started(progress_id, metadata, client.profile)
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
247 session = { #  session data, can be used by importers
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
248 "root_service": pubsub_service,
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
249 "root_node": pubsub_node,
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
250 }
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
251 self.recursive_import(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
252 client,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
253 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
254 items_import_data,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
255 progress_id,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
256 session,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
257 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
258 None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
259 pubsub_service,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
260 pubsub_node,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
261 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
262 defer.returnValue(progress_id)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
263
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
264 @defer.inlineCallbacks
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
265 def recursive_import(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
266 self,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
267 client,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
268 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
269 items_import_data,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
270 progress_id,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
271 session,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
272 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
273 return_data=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
274 service=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
275 node=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
276 depth=0,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
277 ):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
278 """Do the import recursively
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
279
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
280 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
281 @param items_import_data(iterable): iterable of data as specified in [register]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
282 @param progress_id(unicode): id of progression
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
283 @param session(dict): data for this import session
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
284 can be used by importer so store any useful data
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
285 "root_service" and "root_node" are set to the main pubsub service and node of the import
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
286 @param options(dict): import options
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
287 @param return_data(dict): data to return on progress_finished
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
288 @param service(jid.JID, None): PubSub service to use
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
289 @param node(unicode, None): PubSub node to use
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
290 @param depth(int): level of recursion
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
291 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
292 if return_data is None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
293 return_data = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
294 for idx, item_import_data in enumerate(items_import_data):
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
295 item_data = yield import_handler.import_item(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
296 client, item_import_data, session, options, return_data, service, node
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
297 )
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
298 yield import_handler.item_filters(client, item_data, session, options)
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
299 recurse_kwargs = yield import_handler.import_sub_items(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
300 client, item_import_data, item_data, session, options
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
301 )
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
302 yield import_handler.publish_item(client, item_data, service, node, session)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
303
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
304 if recurse_kwargs is not None:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
305 recurse_kwargs["client"] = client
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
306 recurse_kwargs["import_handler"] = import_handler
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
307 recurse_kwargs["progress_id"] = progress_id
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
308 recurse_kwargs["session"] = session
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
309 recurse_kwargs.setdefault("options", options)
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
310 recurse_kwargs["return_data"] = return_data
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
311 recurse_kwargs["depth"] = depth + 1
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
312 log.debug(_("uploading subitems"))
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
313 yield self.recursive_import(**recurse_kwargs)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
314
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
315 if depth == 0:
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
316 client._import[import_handler.name][progress_id]["position"] = str(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
317 idx + 1
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
318 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
319
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
320 if depth == 0:
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
321 self.host.bridge.progress_finished(progress_id, return_data, client.profile)
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
322 self.host.remove_progress_cb(progress_id, client.profile)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
323 del client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
324
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
325 def register(self, import_handler, name, callback, short_desc="", long_desc=""):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
326 """Register an Importer method
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
327
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
328 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
329 @param callback(callable): method to call:
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
330 the signature must be (client, location, options) (cf. [do_import])
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
331 the importer must return a tuple with (items_import_data, items_count)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
332 items_import_data(iterable[dict]) data specific to specialized importer
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
333 cf. import_item docstring of specialized importer for details
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
334 items_count (int, None) indicate the total number of items (without subitems)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
335 useful to display a progress indicator when the iterator is a generator
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
336 use None if you can't guess the total number of items
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
337 @param short_desc(unicode): one line description of the importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
338 @param long_desc(unicode): long description of the importer, its options, etc.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
339 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
340 name = name.lower()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
341 if name in import_handler.importers:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
342 raise exceptions.ConflictError(
4270
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
343 _("An {handler_name} importer with the name {name} already exist").format(
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
344 handler_name=import_handler.name, name=name
0d7bb4df2343 Reformatted code base using black.
Goffi <goffi@goffi.org>
parents: 4071
diff changeset
345 )
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
346 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
347 import_handler.importers[name] = Importer(callback, short_desc, long_desc)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
348
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
349 def unregister(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
350 del import_handler.importers[name]