Mercurial > libervia-backend
annotate sat/plugins/plugin_import.py @ 2617:81b70eeb710f
quick_frontend(contact list): refactored update:
update is now called with appropriate constant value (C.UPDATE_ADD, C.UPDATE_DELETE, C.UPDATE_MODIFY and so on) when a widget change visibility according to current options.
Before it was linked to cache only (C.UPDATE_ADD was only called when contact was first added to cache).
This make widget handling in frontends more easy.
Renamed entityToShow to entityVisible, which seems to correspond better.
Started reducing lines lenght to 90 chars as a test. May become the new coding style soon.
author | Goffi <goffi@goffi.org> |
---|---|
date | Sun, 24 Jun 2018 21:59:29 +0200 |
parents | 26edcf3a30eb |
children | 56f94936df1e |
rev | line source |
---|---|
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
1 #!/usr/bin/env python2 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
2 # -*- coding: utf-8 -*- |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
3 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
4 # SàT plugin for generic data import handling |
2483 | 5 # Copyright (C) 2009-2018 Jérôme Poisson (goffi@goffi.org) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # This program is free software: you can redistribute it and/or modify |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # it under the terms of the GNU Affero General Public License as published by |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # the Free Software Foundation, either version 3 of the License, or |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 # (at your option) any later version. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # This program is distributed in the hope that it will be useful, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 # GNU Affero General Public License for more details. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # You should have received a copy of the GNU Affero General Public License |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
19 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
20 from sat.core.i18n import _ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
21 from sat.core.constants import Const as C |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
22 from sat.core.log import getLogger |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
23 log = getLogger(__name__) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
24 from twisted.internet import defer |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
25 from sat.core import exceptions |
2436
b52e78cc86ed
plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
26 from twisted.words.protocols.jabber import jid |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
27 from functools import partial |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
28 import collections |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
29 import uuid |
2396
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2390
diff
changeset
|
30 import json |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
31 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
32 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
33 PLUGIN_INFO = { |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
34 C.PI_NAME: "import", |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 C.PI_IMPORT_NAME: "IMPORT", |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
36 C.PI_TYPE: C.PLUG_TYPE_IMPORT, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
37 C.PI_DEPENDENCIES: [], |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
38 C.PI_MAIN: "ImportPlugin", |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
39 C.PI_HANDLER: "no", |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
40 C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers""") |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
41 } |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
42 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
43 Importer = collections.namedtuple('Importer', ('callback', 'short_desc', 'long_desc')) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
44 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
45 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
46 class ImportPlugin(object): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
47 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
48 def __init__(self, host): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
49 log.info(_("plugin Import initialization")) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
50 self.host = host |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
51 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
52 def initialize(self, import_handler, name): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
53 """Initialize a specialized import handler |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
54 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
55 @param import_handler(object): specialized import handler instance |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
56 must have the following methods: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
57 - importItem: import a single main item (i.e. prepare data for publishing) |
2390
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
58 - importSubitems: import sub items (i.e. items linked to main item, e.g. comments). |
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
59 Must return a dict with kwargs for recursiveImport if items are to be imported recursively. |
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
60 At least "items_import_data", "service" and "node" keys must be provided. |
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
61 if None is returned, no recursion will be done to import subitems, but import can still be done directly by the method. |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
62 - publishItem: actualy publish an item |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
63 - itemFilters: modify item according to options |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
64 @param name(unicode): import handler name |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
65 """ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
66 assert name == name.lower().strip() |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
67 log.info(_(u'initializing {name} import handler').format(name=name)) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
68 import_handler.name = name |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
69 import_handler.register = partial(self.register, import_handler) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
70 import_handler.unregister = partial(self.unregister, import_handler) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
71 import_handler.importers = {} |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
72 def _import(name, location, options, pubsub_service, pubsub_node, profile): |
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
73 return self._doImport(import_handler, name, location, options, pubsub_service, pubsub_node, profile) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
74 def _importList(): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
75 return self.listImporters(import_handler) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
76 def _importDesc(name): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
77 return self.getDescription(import_handler, name) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
78 |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
79 self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}sss', out_sign='s', method=_import, async=True) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
80 self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
81 self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
82 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
83 def getProgress(self, import_handler, progress_id, profile): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
84 client = self.host.getClient(profile) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
85 return client._import[import_handler.name][progress_id] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
86 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
87 def listImporters(self, import_handler): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
88 importers = import_handler.importers.keys() |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
89 importers.sort() |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
90 return [(name, import_handler.importers[name].short_desc) for name in import_handler.importers] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
91 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
92 def getDescription(self, import_handler, name): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
93 """Return import short and long descriptions |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
94 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
95 @param name(unicode): importer name |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
96 @return (tuple[unicode,unicode]): short and long description |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
97 """ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
98 try: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
99 importer = import_handler.importers[name] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
100 except KeyError: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
101 raise exceptions.NotFound(u"{handler_name} importer not found [{name}]".format( |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
102 handler_name = import_handler.name, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
103 name = name)) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
104 else: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
105 return importer.short_desc, importer.long_desc |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
106 |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
107 def _doImport(self, import_handler, name, location, options, pubsub_service='', pubsub_node='', profile=C.PROF_KEY_NONE): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
108 client = self.host.getClient(profile) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
109 options = {key: unicode(value) for key, value in options.iteritems()} |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
110 for option in import_handler.BOOL_OPTIONS: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
111 try: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
112 options[option] = C.bool(options[option]) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
113 except KeyError: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
114 pass |
2396
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2390
diff
changeset
|
115 for option in import_handler.JSON_OPTIONS: |
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2390
diff
changeset
|
116 try: |
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2390
diff
changeset
|
117 options[option] = json.loads(options[option]) |
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2390
diff
changeset
|
118 except ValueError: |
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2390
diff
changeset
|
119 raise exceptions.DataError(_(u'invalid json option: {name}').format(name=option)) |
2436
b52e78cc86ed
plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
120 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None |
b52e78cc86ed
plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
121 return self.doImport(client, import_handler, unicode(name), unicode(location), options, pubsub_service, pubsub_node or None) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
122 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
123 @defer.inlineCallbacks |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
124 def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None, pubsub_node=None): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
125 """Import data |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
126 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
127 @param import_handler(object): instance of the import handler |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
128 @param name(unicode): name of the importer |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
129 @param location(unicode): location of the data to import |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
130 can be an url, a file path, or anything which make sense |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
131 check importer description for more details |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
132 @param options(dict, None): extra options. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
133 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
134 None to use profile's server |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
135 @param pubsub_node(unicode, None): PubSub node to use |
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
136 None to use importer's default node |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
137 @return (unicode): progress id |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
138 """ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
139 if options is None: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
140 options = {} |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
141 else: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
142 for opt_name, opt_default in import_handler.OPT_DEFAULTS.iteritems(): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
143 # we want a filled options dict, with all empty or False values removed |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
144 try: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
145 value =options[opt_name] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
146 except KeyError: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
147 if opt_default: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
148 options[opt_name] = opt_default |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
149 else: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
150 if not value: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
151 del options[opt_name] |
2390
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
152 |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
153 try: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
154 importer = import_handler.importers[name] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
155 except KeyError: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
156 raise exceptions.NotFound(u"Importer [{}] not found".format(name)) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
157 items_import_data, items_count = yield importer.callback(client, location, options) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
158 progress_id = unicode(uuid.uuid4()) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
159 try: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
160 _import = client._import |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
161 except AttributeError: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
162 _import = client._import = {} |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
163 progress_data = _import.setdefault(import_handler.name, {}) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
164 progress_data[progress_id] = {u'position': '0'} |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
165 if items_count is not None: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
166 progress_data[progress_id]['size'] = unicode(items_count) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
167 metadata = {'name': u'{}: {}'.format(name, location), |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
168 'direction': 'out', |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
169 'type': import_handler.name.upper() + '_IMPORT' |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
170 } |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
171 self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
172 self.host.bridge.progressStarted(progress_id, metadata, client.profile) |
2390
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
173 session = { # session data, can be used by importers |
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
174 u'root_service': pubsub_service, |
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
175 u'root_node': pubsub_node |
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
176 } |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
177 self.recursiveImport(client, import_handler, items_import_data, progress_id, session, options, None, pubsub_service, pubsub_node) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
178 defer.returnValue(progress_id) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
179 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
180 @defer.inlineCallbacks |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
181 def recursiveImport(self, client, import_handler, items_import_data, progress_id, session, options, return_data=None, service=None, node=None, depth=0): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
182 """Do the import recursively |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
183 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
184 @param import_handler(object): instance of the import handler |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
185 @param items_import_data(iterable): iterable of data as specified in [register] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
186 @param progress_id(unicode): id of progression |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
187 @param session(dict): data for this import session |
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
188 can be used by importer so store any useful data |
2390
f57a8eaec8ed
plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
189 "root_service" and "root_node" are set to the main pubsub service and node of the import |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
190 @param options(dict): import options |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
191 @param return_data(dict): data to return on progressFinished |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
192 @param service(jid.JID, None): PubSub service to use |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
193 @param node(unicode, None): PubSub node to use |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
194 @param depth(int): level of recursion |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
195 """ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
196 if return_data is None: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
197 return_data = {} |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
198 for idx, item_import_data in enumerate(items_import_data): |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
199 item_data = yield import_handler.importItem(client, item_import_data, session, options, return_data, service, node) |
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
200 yield import_handler.itemFilters(client, item_data, session, options) |
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
201 recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, session, options) |
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
202 yield import_handler.publishItem(client, item_data, service, node, session) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
203 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
204 if recurse_kwargs is not None: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
205 recurse_kwargs['client'] = client |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
206 recurse_kwargs['import_handler'] = import_handler |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
207 recurse_kwargs['progress_id'] = progress_id |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
208 recurse_kwargs['session'] = session |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
209 recurse_kwargs.setdefault('options', options) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
210 recurse_kwargs['return_data'] = return_data |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
211 recurse_kwargs['depth'] = depth + 1 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
212 log.debug(_(u"uploading subitems")) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
213 yield self.recursiveImport(**recurse_kwargs) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
214 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
215 if depth == 0: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
216 client._import[import_handler.name][progress_id]['position'] = unicode(idx+1) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
217 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
218 if depth == 0: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
219 self.host.bridge.progressFinished(progress_id, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
220 return_data, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
221 client.profile) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
222 self.host.removeProgressCb(progress_id, client.profile) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
223 del client._import[import_handler.name][progress_id] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
224 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
225 def register(self, import_handler, name, callback, short_desc='', long_desc=''): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
226 """Register an Importer method |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
227 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
228 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
229 @param callback(callable): method to call: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
230 the signature must be (client, location, options) (cf. [doImport]) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
231 the importer must return a tuple with (items_import_data, items_count) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
232 items_import_data(iterable[dict]) data specific to specialized importer |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
233 cf. importItem docstring of specialized importer for details |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
234 items_count (int, None) indicate the total number of items (without subitems) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
235 useful to display a progress indicator when the iterator is a generator |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
236 use None if you can't guess the total number of items |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
237 @param short_desc(unicode): one line description of the importer |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
238 @param long_desc(unicode): long description of the importer, its options, etc. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
239 """ |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
240 name = name.lower() |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
241 if name in import_handler.importers: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
242 raise exceptions.ConflictError(_(u"An {handler_name} importer with the name {name} already exist").format( |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
243 handler_name = import_handler.name, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
244 name = name)) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
245 import_handler.importers[name] = Importer(callback, short_desc, long_desc) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
246 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
247 def unregister(self, import_handler, name): |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
248 del import_handler.importers[name] |