Mercurial > libervia-backend
annotate sat/plugins/plugin_import.py @ 2876:84f82d014d06
jp (blog/set): fixed use of --item argument
author | Goffi <goffi@goffi.org> |
---|---|
date | Mon, 25 Mar 2019 18:40:10 +0100 |
parents | 003b8b4b56a7 |
children | ab2696e34d29 |
rev | line source |
---|---|
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
1 #!/usr/bin/env python2 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
2 # -*- coding: utf-8 -*- |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
3 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
4 # SàT plugin for generic data import handling |
2771 | 5 # Copyright (C) 2009-2019 Jérôme Poisson (goffi@goffi.org) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # This program is free software: you can redistribute it and/or modify |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # it under the terms of the GNU Affero General Public License as published by |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # the Free Software Foundation, either version 3 of the License, or |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 # (at your option) any later version. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # This program is distributed in the hope that it will be useful, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 # GNU Affero General Public License for more details. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # You should have received a copy of the GNU Affero General Public License |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
19 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
20 from sat.core.i18n import _ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
21 from sat.core.constants import Const as C |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
22 from sat.core.log import getLogger |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
23 |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
24 log = getLogger(__name__) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
25 from twisted.internet import defer |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
26 from sat.core import exceptions |
2436
b52e78cc86ed
plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
27 from twisted.words.protocols.jabber import jid |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
28 from functools import partial |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
29 import collections |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
30 import uuid |
2396
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2390
diff
changeset
|
31 import json |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
32 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
33 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
34 PLUGIN_INFO = { |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 C.PI_NAME: "import", |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
36 C.PI_IMPORT_NAME: "IMPORT", |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
37 C.PI_TYPE: C.PLUG_TYPE_IMPORT, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
38 C.PI_DEPENDENCIES: [], |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
39 C.PI_MAIN: "ImportPlugin", |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
40 C.PI_HANDLER: "no", |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
41 C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers"""), |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
42 } |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
43 |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
44 Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc")) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
45 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
46 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
47 class ImportPlugin(object): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
48 def __init__(self, host): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
49 log.info(_("plugin Import initialization")) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
50 self.host = host |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
51 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
52 def initialize(self, import_handler, name): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
53 """Initialize a specialized import handler |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
54 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
55 @param import_handler(object): specialized import handler instance |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
56 must have the following methods: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
57 - importItem: import a single main item (i.e. prepare data for publishing) |
2390
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
58 - importSubitems: import sub items (i.e. items linked to main item, e.g. comments). |
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
59 Must return a dict with kwargs for recursiveImport if items are to be imported recursively. |
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
60 At least "items_import_data", "service" and "node" keys must be provided. |
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
61 if None is returned, no recursion will be done to import subitems, but import can still be done directly by the method. |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
62 - publishItem: actualy publish an item |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
63 - itemFilters: modify item according to options |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
64 @param name(unicode): import handler name |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
65 """ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
66 assert name == name.lower().strip() |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
67 log.info(_(u"initializing {name} import handler").format(name=name)) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
68 import_handler.name = name |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
69 import_handler.register = partial(self.register, import_handler) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
70 import_handler.unregister = partial(self.unregister, import_handler) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
71 import_handler.importers = {} |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
72 |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
73 def _import(name, location, options, pubsub_service, pubsub_node, profile): |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
74 return self._doImport( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
75 import_handler, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
76 name, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
77 location, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
78 options, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
79 pubsub_service, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
80 pubsub_node, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
81 profile, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
82 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
83 |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
84 def _importList(): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
85 return self.listImporters(import_handler) |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
86 |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
87 def _importDesc(name): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
88 return self.getDescription(import_handler, name) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
89 |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
90 self.host.bridge.addMethod( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
91 name + "Import", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
92 ".plugin", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
93 in_sign="ssa{ss}sss", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
94 out_sign="s", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
95 method=_import, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
96 async=True, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
97 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
98 self.host.bridge.addMethod( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
99 name + "ImportList", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
100 ".plugin", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
101 in_sign="", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
102 out_sign="a(ss)", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
103 method=_importList, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
104 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
105 self.host.bridge.addMethod( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
106 name + "ImportDesc", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
107 ".plugin", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
108 in_sign="s", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
109 out_sign="(ss)", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
110 method=_importDesc, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
111 ) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
112 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
113 def getProgress(self, import_handler, progress_id, profile): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
114 client = self.host.getClient(profile) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
115 return client._import[import_handler.name][progress_id] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
116 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
117 def listImporters(self, import_handler): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
118 importers = import_handler.importers.keys() |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
119 importers.sort() |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
120 return [ |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
121 (name, import_handler.importers[name].short_desc) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
122 for name in import_handler.importers |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
123 ] |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
124 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
125 def getDescription(self, import_handler, name): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
126 """Return import short and long descriptions |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
127 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
128 @param name(unicode): importer name |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
129 @return (tuple[unicode,unicode]): short and long description |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
130 """ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
131 try: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
132 importer = import_handler.importers[name] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
133 except KeyError: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
134 raise exceptions.NotFound( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
135 u"{handler_name} importer not found [{name}]".format( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
136 handler_name=import_handler.name, name=name |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
137 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
138 ) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
139 else: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
140 return importer.short_desc, importer.long_desc |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
141 |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
142 def _doImport( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
143 self, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
144 import_handler, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
145 name, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
146 location, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
147 options, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
148 pubsub_service="", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
149 pubsub_node="", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
150 profile=C.PROF_KEY_NONE, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
151 ): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
152 client = self.host.getClient(profile) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
153 options = {key: unicode(value) for key, value in options.iteritems()} |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
154 for option in import_handler.BOOL_OPTIONS: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
155 try: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
156 options[option] = C.bool(options[option]) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
157 except KeyError: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
158 pass |
2396
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2390
diff
changeset
|
159 for option in import_handler.JSON_OPTIONS: |
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2390
diff
changeset
|
160 try: |
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2390
diff
changeset
|
161 options[option] = json.loads(options[option]) |
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2390
diff
changeset
|
162 except ValueError: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
163 raise exceptions.DataError( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
164 _(u"invalid json option: {name}").format(name=option) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
165 ) |
2436
b52e78cc86ed
plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
166 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
167 return self.doImport( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
168 client, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
169 import_handler, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
170 unicode(name), |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
171 unicode(location), |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
172 options, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
173 pubsub_service, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
174 pubsub_node or None, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
175 ) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
176 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
177 @defer.inlineCallbacks |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
178 def doImport( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
179 self, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
180 client, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
181 import_handler, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
182 name, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
183 location, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
184 options=None, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
185 pubsub_service=None, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
186 pubsub_node=None, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
187 ): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
188 """Import data |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
189 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
190 @param import_handler(object): instance of the import handler |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
191 @param name(unicode): name of the importer |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
192 @param location(unicode): location of the data to import |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
193 can be an url, a file path, or anything which make sense |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
194 check importer description for more details |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
195 @param options(dict, None): extra options. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
196 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
197 None to use profile's server |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
198 @param pubsub_node(unicode, None): PubSub node to use |
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
199 None to use importer's default node |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
200 @return (unicode): progress id |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
201 """ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
202 if options is None: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
203 options = {} |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
204 else: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
205 for opt_name, opt_default in import_handler.OPT_DEFAULTS.iteritems(): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
206 # we want a filled options dict, with all empty or False values removed |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
207 try: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
208 value = options[opt_name] |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
209 except KeyError: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
210 if opt_default: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
211 options[opt_name] = opt_default |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
212 else: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
213 if not value: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
214 del options[opt_name] |
2390
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
215 |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
216 try: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
217 importer = import_handler.importers[name] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
218 except KeyError: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
219 raise exceptions.NotFound(u"Importer [{}] not found".format(name)) |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
220 items_import_data, items_count = yield importer.callback( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
221 client, location, options |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
222 ) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
223 progress_id = unicode(uuid.uuid4()) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
224 try: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
225 _import = client._import |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
226 except AttributeError: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
227 _import = client._import = {} |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
228 progress_data = _import.setdefault(import_handler.name, {}) |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
229 progress_data[progress_id] = {u"position": "0"} |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
230 if items_count is not None: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
231 progress_data[progress_id]["size"] = unicode(items_count) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
232 metadata = { |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
233 "name": u"{}: {}".format(name, location), |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
234 "direction": "out", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
235 "type": import_handler.name.upper() + "_IMPORT", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
236 } |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
237 self.host.registerProgressCb( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
238 progress_id, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
239 partial(self.getProgress, import_handler), |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
240 metadata, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
241 profile=client.profile, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
242 ) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
243 self.host.bridge.progressStarted(progress_id, metadata, client.profile) |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
244 session = { # session data, can be used by importers |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
245 u"root_service": pubsub_service, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
246 u"root_node": pubsub_node, |
2390
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
247 } |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
248 self.recursiveImport( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
249 client, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
250 import_handler, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
251 items_import_data, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
252 progress_id, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
253 session, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
254 options, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
255 None, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
256 pubsub_service, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
257 pubsub_node, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
258 ) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
259 defer.returnValue(progress_id) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
260 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
261 @defer.inlineCallbacks |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
262 def recursiveImport( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
263 self, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
264 client, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
265 import_handler, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
266 items_import_data, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
267 progress_id, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
268 session, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
269 options, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
270 return_data=None, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
271 service=None, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
272 node=None, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
273 depth=0, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
274 ): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
275 """Do the import recursively |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
276 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
277 @param import_handler(object): instance of the import handler |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
278 @param items_import_data(iterable): iterable of data as specified in [register] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
279 @param progress_id(unicode): id of progression |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
280 @param session(dict): data for this import session |
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
281 can be used by importer so store any useful data |
2390
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
282 "root_service" and "root_node" are set to the main pubsub service and node of the import |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
283 @param options(dict): import options |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
284 @param return_data(dict): data to return on progressFinished |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
285 @param service(jid.JID, None): PubSub service to use |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
286 @param node(unicode, None): PubSub node to use |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
287 @param depth(int): level of recursion |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
288 """ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
289 if return_data is None: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
290 return_data = {} |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
291 for idx, item_import_data in enumerate(items_import_data): |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
292 item_data = yield import_handler.importItem( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
293 client, item_import_data, session, options, return_data, service, node |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
294 ) |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
295 yield import_handler.itemFilters(client, item_data, session, options) |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
296 recurse_kwargs = yield import_handler.importSubItems( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
297 client, item_import_data, item_data, session, options |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
298 ) |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
299 yield import_handler.publishItem(client, item_data, service, node, session) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
300 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
301 if recurse_kwargs is not None: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
302 recurse_kwargs["client"] = client |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
303 recurse_kwargs["import_handler"] = import_handler |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
304 recurse_kwargs["progress_id"] = progress_id |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
305 recurse_kwargs["session"] = session |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
306 recurse_kwargs.setdefault("options", options) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
307 recurse_kwargs["return_data"] = return_data |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
308 recurse_kwargs["depth"] = depth + 1 |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
309 log.debug(_(u"uploading subitems")) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
310 yield self.recursiveImport(**recurse_kwargs) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
311 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
312 if depth == 0: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
313 client._import[import_handler.name][progress_id]["position"] = unicode( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
314 idx + 1 |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
315 ) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
316 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
317 if depth == 0: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
318 self.host.bridge.progressFinished(progress_id, return_data, client.profile) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
319 self.host.removeProgressCb(progress_id, client.profile) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
320 del client._import[import_handler.name][progress_id] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
321 |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
322 def register(self, import_handler, name, callback, short_desc="", long_desc=""): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
323 """Register an Importer method |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
324 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
325 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
326 @param callback(callable): method to call: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
327 the signature must be (client, location, options) (cf. [doImport]) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
328 the importer must return a tuple with (items_import_data, items_count) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
329 items_import_data(iterable[dict]) data specific to specialized importer |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
330 cf. importItem docstring of specialized importer for details |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
331 items_count (int, None) indicate the total number of items (without subitems) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
332 useful to display a progress indicator when the iterator is a generator |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
333 use None if you can't guess the total number of items |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
334 @param short_desc(unicode): one line description of the importer |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
335 @param long_desc(unicode): long description of the importer, its options, etc. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
336 """ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
337 name = name.lower() |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
338 if name in import_handler.importers: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
339 raise exceptions.ConflictError( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
340 _( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
341 u"An {handler_name} importer with the name {name} already exist" |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
342 ).format(handler_name=import_handler.name, name=name) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
343 ) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
344 import_handler.importers[name] = Importer(callback, short_desc, long_desc) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
345 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
346 def unregister(self, import_handler, name): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
347 del import_handler.importers[name] |