Mercurial > libervia-backend

annotate libervia/backend/plugins/plugin_import.py @ 4219:1b5cf2ee1d86

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
plugin XEP-0384, XEP-0391: download missing devices list: when a peer jid was not in our roster, devices list was not retrieved, resulting in failed en/decryption. This patch does check it and download missing devices list in necessary. There is no subscription managed yet, so the list won't be updated in case of new devices, this should be addressed at some point.
author Goffi <goffi@goffi.org>
date Tue, 05 Mar 2024 17:31:36 +0100
parents 4b842c1fb686
children 0d7bb4df2343
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
1 #!/usr/bin/env python3
3137
559a625a236b fixed shebangs
Goffi <goffi@goffi.org>
parents: 3136
diff changeset
2
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
3
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
4 # SàT plugin for generic data import handling
3479
be6d91572633 date update
Goffi <goffi@goffi.org>
parents: 3137
diff changeset
5 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
6
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
7 # This program is free software: you can redistribute it and/or modify
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
8 # it under the terms of the GNU Affero General Public License as published by
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
9 # the Free Software Foundation, either version 3 of the License, or
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
10 # (at your option) any later version.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
11
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
12 # This program is distributed in the hope that it will be useful,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
15 # GNU Affero General Public License for more details.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
16
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
17 # You should have received a copy of the GNU Affero General Public License
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
19
4071
4b842c1fb686 refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents: 4037
diff changeset
20 from libervia.backend.core.i18n import _
4b842c1fb686 refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents: 4037
diff changeset
21 from libervia.backend.core.constants import Const as C
4b842c1fb686 refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents: 4037
diff changeset
22 from libervia.backend.core.log import getLogger
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
23
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
24 log = getLogger(__name__)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
25 from twisted.internet import defer
4071
4b842c1fb686 refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents: 4037
diff changeset
26 from libervia.backend.core import exceptions
2436
b52e78cc86ed plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents: 2414
diff changeset
27 from twisted.words.protocols.jabber import jid
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
28 from functools import partial
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
29 import collections
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
30 import uuid
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
31 import json
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
32
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
33
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
34 PLUGIN_INFO = {
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
35 C.PI_NAME: "import",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
36 C.PI_IMPORT_NAME: "IMPORT",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
37 C.PI_TYPE: C.PLUG_TYPE_IMPORT,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
38 C.PI_DEPENDENCIES: [],
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
39 C.PI_MAIN: "ImportPlugin",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
40 C.PI_HANDLER: "no",
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
41 C.PI_DESCRIPTION: _("""Generic import plugin, base for specialized importers"""),
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
42 }
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
43
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
44 Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc"))
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
45
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
46
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
47 class ImportPlugin(object):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
48 def __init__(self, host):
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
49 log.info(_("plugin import initialization"))
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
50 self.host = host
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
51
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
52 def initialize(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
53 """Initialize a specialized import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
54
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
55 @param import_handler(object): specialized import handler instance
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
56 must have the following methods:
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
57 - import_item: import a single main item (i.e. prepare data for publishing)
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
58 - importSubitems: import sub items (i.e. items linked to main item, e.g. comments).
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
59 Must return a dict with kwargs for recursive_import if items are to be imported recursively.
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
60 At least "items_import_data", "service" and "node" keys must be provided.
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
61 if None is returned, no recursion will be done to import subitems, but import can still be done directly by the method.
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
62 - publish_item: actualy publish an item
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
63 - item_filters: modify item according to options
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
64 @param name(unicode): import handler name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
65 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
66 assert name == name.lower().strip()
3568
04283582966f core, frontends: fix invalid translatable strings.
Goffi <goffi@goffi.org>
parents: 3479
diff changeset
67 log.info(_("initializing {name} import handler").format(name=name))
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
68 import_handler.name = name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
69 import_handler.register = partial(self.register, import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
70 import_handler.unregister = partial(self.unregister, import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
71 import_handler.importers = {}
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
72
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
73 def _import(name, location, options, pubsub_service, pubsub_node, profile):
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
74 return self._do_import(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
75 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
76 name,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
77 location,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
78 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
79 pubsub_service,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
80 pubsub_node,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
81 profile,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
82 )
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
83
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
84 def _import_list():
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
85 return self.list_importers(import_handler)
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
86
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
87 def _import_desc(name):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
88 return self.getDescription(import_handler, name)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
89
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
90 self.host.bridge.add_method(
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
91 name + "import",
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
92 ".plugin",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
93 in_sign="ssa{ss}sss",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
94 out_sign="s",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
95 method=_import,
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
96 async_=True,
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
97 )
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
98 self.host.bridge.add_method(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
99 name + "ImportList",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
100 ".plugin",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
101 in_sign="",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
102 out_sign="a(ss)",
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
103 method=_import_list,
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
104 )
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
105 self.host.bridge.add_method(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
106 name + "ImportDesc",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
107 ".plugin",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
108 in_sign="s",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
109 out_sign="(ss)",
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
110 method=_import_desc,
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
111 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
112
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
113 def get_progress(self, import_handler, progress_id, profile):
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
114 client = self.host.get_client(profile)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
115 return client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
116
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
117 def list_importers(self, import_handler):
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
118 importers = list(import_handler.importers.keys())
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
119 importers.sort()
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
120 return [
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
121 (name, import_handler.importers[name].short_desc)
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
122 for name in import_handler.importers
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
123 ]
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
124
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
125 def getDescription(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
126 """Return import short and long descriptions
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
127
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
128 @param name(unicode): importer name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
129 @return (tuple[unicode,unicode]): short and long description
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
130 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
131 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
132 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
133 except KeyError:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
134 raise exceptions.NotFound(
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
135 "{handler_name} importer not found [{name}]".format(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
136 handler_name=import_handler.name, name=name
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
137 )
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
138 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
139 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
140 return importer.short_desc, importer.long_desc
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
141
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
142 def _do_import(self, import_handler, name, location, options, pubsub_service="",
3040
fee60f17ebac jp: jp asyncio port:
Goffi <goffi@goffi.org>
parents: 3028
diff changeset
143 pubsub_node="", profile=C.PROF_KEY_NONE):
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
144 client = self.host.get_client(profile)
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
145 options = {key: str(value) for key, value in options.items()}
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
146 for option in import_handler.BOOL_OPTIONS:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
147 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
148 options[option] = C.bool(options[option])
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
149 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
150 pass
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
151 for option in import_handler.JSON_OPTIONS:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
152 try:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
153 options[option] = json.loads(options[option])
3040
fee60f17ebac jp: jp asyncio port:
Goffi <goffi@goffi.org>
parents: 3028
diff changeset
154 except KeyError:
fee60f17ebac jp: jp asyncio port:
Goffi <goffi@goffi.org>
parents: 3028
diff changeset
155 pass
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
156 except ValueError:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
157 raise exceptions.DataError(
3568
04283582966f core, frontends: fix invalid translatable strings.
Goffi <goffi@goffi.org>
parents: 3479
diff changeset
158 _("invalid json option: {option}").format(option=option)
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
159 )
2436
b52e78cc86ed plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents: 2414
diff changeset
160 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
161 return self.do_import(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
162 client,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
163 import_handler,
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
164 str(name),
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
165 str(location),
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
166 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
167 pubsub_service,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
168 pubsub_node or None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
169 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
170
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
171 @defer.inlineCallbacks
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
172 def do_import(self, client, import_handler, name, location, options=None,
3040
fee60f17ebac jp: jp asyncio port:
Goffi <goffi@goffi.org>
parents: 3028
diff changeset
173 pubsub_service=None, pubsub_node=None,):
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
174 """import data
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
175
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
176 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
177 @param name(unicode): name of the importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
178 @param location(unicode): location of the data to import
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
179 can be an url, a file path, or anything which make sense
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
180 check importer description for more details
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
181 @param options(dict, None): extra options.
3040
fee60f17ebac jp: jp asyncio port:
Goffi <goffi@goffi.org>
parents: 3028
diff changeset
182 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be
fee60f17ebac jp: jp asyncio port:
Goffi <goffi@goffi.org>
parents: 3028
diff changeset
183 imported.
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
184 None to use profile's server
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
185 @param pubsub_node(unicode, None): PubSub node to use
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
186 None to use importer's default node
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
187 @return (unicode): progress id
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
188 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
189 if options is None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
190 options = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
191 else:
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
192 for opt_name, opt_default in import_handler.OPT_DEFAULTS.items():
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
193 # we want a filled options dict, with all empty or False values removed
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
194 try:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
195 value = options[opt_name]
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
196 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
197 if opt_default:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
198 options[opt_name] = opt_default
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
199 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
200 if not value:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
201 del options[opt_name]
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
202
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
203 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
204 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
205 except KeyError:
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
206 raise exceptions.NotFound("Importer [{}] not found".format(name))
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
207 items_import_data, items_count = yield importer.callback(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
208 client, location, options
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
209 )
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
210 progress_id = str(uuid.uuid4())
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
211 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
212 _import = client._import
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
213 except AttributeError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
214 _import = client._import = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
215 progress_data = _import.setdefault(import_handler.name, {})
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
216 progress_data[progress_id] = {"position": "0"}
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
217 if items_count is not None:
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
218 progress_data[progress_id]["size"] = str(items_count)
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
219 metadata = {
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
220 "name": "{}: {}".format(name, location),
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
221 "direction": "out",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
222 "type": import_handler.name.upper() + "_IMPORT",
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
223 }
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
224 self.host.register_progress_cb(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
225 progress_id,
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
226 partial(self.get_progress, import_handler),
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
227 metadata,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
228 profile=client.profile,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
229 )
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
230 self.host.bridge.progress_started(progress_id, metadata, client.profile)
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
231 session = { #  session data, can be used by importers
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
232 "root_service": pubsub_service,
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
233 "root_node": pubsub_node,
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
234 }
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
235 self.recursive_import(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
236 client,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
237 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
238 items_import_data,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
239 progress_id,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
240 session,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
241 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
242 None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
243 pubsub_service,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
244 pubsub_node,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
245 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
246 defer.returnValue(progress_id)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
247
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
248 @defer.inlineCallbacks
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
249 def recursive_import(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
250 self,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
251 client,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
252 import_handler,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
253 items_import_data,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
254 progress_id,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
255 session,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
256 options,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
257 return_data=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
258 service=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
259 node=None,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
260 depth=0,
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
261 ):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
262 """Do the import recursively
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
263
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
264 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
265 @param items_import_data(iterable): iterable of data as specified in [register]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
266 @param progress_id(unicode): id of progression
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
267 @param session(dict): data for this import session
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
268 can be used by importer so store any useful data
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
269 "root_service" and "root_node" are set to the main pubsub service and node of the import
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
270 @param options(dict): import options
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
271 @param return_data(dict): data to return on progress_finished
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
272 @param service(jid.JID, None): PubSub service to use
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
273 @param node(unicode, None): PubSub node to use
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
274 @param depth(int): level of recursion
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
275 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
276 if return_data is None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
277 return_data = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
278 for idx, item_import_data in enumerate(items_import_data):
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
279 item_data = yield import_handler.import_item(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
280 client, item_import_data, session, options, return_data, service, node
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
281 )
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
282 yield import_handler.item_filters(client, item_data, session, options)
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
283 recurse_kwargs = yield import_handler.import_sub_items(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
284 client, item_import_data, item_data, session, options
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
285 )
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
286 yield import_handler.publish_item(client, item_data, service, node, session)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
287
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
288 if recurse_kwargs is not None:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
289 recurse_kwargs["client"] = client
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
290 recurse_kwargs["import_handler"] = import_handler
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
291 recurse_kwargs["progress_id"] = progress_id
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
292 recurse_kwargs["session"] = session
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
293 recurse_kwargs.setdefault("options", options)
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
294 recurse_kwargs["return_data"] = return_data
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
295 recurse_kwargs["depth"] = depth + 1
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
296 log.debug(_("uploading subitems"))
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
297 yield self.recursive_import(**recurse_kwargs)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
298
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
299 if depth == 0:
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
300 client._import[import_handler.name][progress_id]["position"] = str(
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
301 idx + 1
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
302 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
303
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
304 if depth == 0:
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
305 self.host.bridge.progress_finished(progress_id, return_data, client.profile)
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
306 self.host.remove_progress_cb(progress_id, client.profile)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
307 del client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
308
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
309 def register(self, import_handler, name, callback, short_desc="", long_desc=""):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
310 """Register an Importer method
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
311
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
312 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
313 @param callback(callable): method to call:
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
314 the signature must be (client, location, options) (cf. [do_import])
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
315 the importer must return a tuple with (items_import_data, items_count)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
316 items_import_data(iterable[dict]) data specific to specialized importer
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3568
diff changeset
317 cf. import_item docstring of specialized importer for details
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
318 items_count (int, None) indicate the total number of items (without subitems)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
319 useful to display a progress indicator when the iterator is a generator
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
320 use None if you can't guess the total number of items
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
321 @param short_desc(unicode): one line description of the importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
322 @param long_desc(unicode): long description of the importer, its options, etc.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
323 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
324 name = name.lower()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
325 if name in import_handler.importers:
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
326 raise exceptions.ConflictError(
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
327 _(
3028
ab2696e34d29 Python 3 port:
Goffi <goffi@goffi.org>
parents: 2771
diff changeset
328 "An {handler_name} importer with the name {name} already exist"
2624
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
329 ).format(handler_name=import_handler.name, name=name)
56f94936df1e code style reformatting using black
Goffi <goffi@goffi.org>
parents: 2562
diff changeset
330 )
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
331 import_handler.importers[name] = Importer(callback, short_desc, long_desc)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
332
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
333 def unregister(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
334 del import_handler.importers[name]