Mercurial > libervia-backend

annotate src/plugins/plugin_import.py @ 2444:30278ea1ca7c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
plugin XEP-0060: added node watching methods to bridge: new methods psNodeWatchAdd and psNodeWatchRemove allows to set a watch for the time of the session on one node, to have a signal called when something change on this node. This signal (psEventRaw) send raw data (raw XML), in opposition to psEvent which is there to send high level data (e.g. parsed blog data). Those method are primarely there to let frontends manage local cache for pubsub nodes.
author Goffi <goffi@goffi.org>
date Sun, 19 Nov 2017 16:51:39 +0100
parents b52e78cc86ed
children 0046283a285d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
1 #!/usr/bin/env python2
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
2 # -*- coding: utf-8 -*-
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
3
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
4 # SàT plugin for generic data import handling
2414
8b37a62336c3 misc: date update (yes it's a bit late :p )
Goffi <goffi@goffi.org>
parents: 2396
diff changeset
5 # Copyright (C) 2009-2017 Jérôme Poisson (goffi@goffi.org)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
6
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
7 # This program is free software: you can redistribute it and/or modify
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
8 # it under the terms of the GNU Affero General Public License as published by
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
9 # the Free Software Foundation, either version 3 of the License, or
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
10 # (at your option) any later version.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
11
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
12 # This program is distributed in the hope that it will be useful,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
15 # GNU Affero General Public License for more details.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
16
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
17 # You should have received a copy of the GNU Affero General Public License
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
19
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
20 from sat.core.i18n import _
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
21 from sat.core.constants import Const as C
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
22 from sat.core.log import getLogger
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
23 log = getLogger(__name__)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
24 from twisted.internet import defer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
25 from sat.core import exceptions
2436
b52e78cc86ed plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents: 2414
diff changeset
26 from twisted.words.protocols.jabber import jid
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
27 from functools import partial
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
28 import collections
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
29 import uuid
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
30 import json
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
31
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
32
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
33 PLUGIN_INFO = {
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
34 C.PI_NAME: "import",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
35 C.PI_IMPORT_NAME: "IMPORT",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
36 C.PI_TYPE: C.PLUG_TYPE_IMPORT,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
37 C.PI_DEPENDENCIES: [],
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
38 C.PI_MAIN: "ImportPlugin",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
39 C.PI_HANDLER: "no",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
40 C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers""")
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
41 }
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
42
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
43 Importer = collections.namedtuple('Importer', ('callback', 'short_desc', 'long_desc'))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
44
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
45
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
46 class ImportPlugin(object):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
47
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
48 def __init__(self, host):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
49 log.info(_("plugin Import initialization"))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
50 self.host = host
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
51
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
52 def initialize(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
53 """Initialize a specialized import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
54
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
55 @param import_handler(object): specialized import handler instance
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
56 must have the following methods:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
57 - importItem: import a single main item (i.e. prepare data for publishing)
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
58 - importSubitems: import sub items (i.e. items linked to main item, e.g. comments).
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
59 Must return a dict with kwargs for recursiveImport if items are to be imported recursively.
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
60 At least "items_import_data", "service" and "node" keys must be provided.
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
61 if None is returned, no recursion will be done to import subitems, but import can still be done directly by the method.
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
62 - publishItem: actualy publish an item
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
63 - itemFilters: modify item according to options
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
64 @param name(unicode): import handler name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
65 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
66 assert name == name.lower().strip()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
67 log.info(_(u'initializing {name} import handler').format(name=name))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
68 import_handler.name = name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
69 import_handler.register = partial(self.register, import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
70 import_handler.unregister = partial(self.unregister, import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
71 import_handler.importers = {}
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
72 def _import(name, location, options, pubsub_service, pubsub_node, profile):
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
73 return self._doImport(import_handler, name, location, options, pubsub_service, pubsub_node, profile)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
74 def _importList():
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
75 return self.listImporters(import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
76 def _importDesc(name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
77 return self.getDescription(import_handler, name)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
78
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
79 self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}sss', out_sign='s', method=_import, async=True)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
80 self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
81 self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
82
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
83 def getProgress(self, import_handler, progress_id, profile):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
84 client = self.host.getClient(profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
85 return client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
86
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
87 def listImporters(self, import_handler):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
88 importers = import_handler.importers.keys()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
89 importers.sort()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
90 return [(name, import_handler.importers[name].short_desc) for name in import_handler.importers]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
91
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
92 def getDescription(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
93 """Return import short and long descriptions
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
94
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
95 @param name(unicode): importer name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
96 @return (tuple[unicode,unicode]): short and long description
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
97 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
98 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
99 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
100 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
101 raise exceptions.NotFound(u"{handler_name} importer not found [{name}]".format(
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
102 handler_name = import_handler.name,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
103 name = name))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
104 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
105 return importer.short_desc, importer.long_desc
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
106
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
107 def _doImport(self, import_handler, name, location, options, pubsub_service='', pubsub_node='', profile=C.PROF_KEY_NONE):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
108 client = self.host.getClient(profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
109 options = {key: unicode(value) for key, value in options.iteritems()}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
110 for option in import_handler.BOOL_OPTIONS:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
111 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
112 options[option] = C.bool(options[option])
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
113 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
114 pass
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
115 for option in import_handler.JSON_OPTIONS:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
116 try:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
117 options[option] = json.loads(options[option])
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
118 except ValueError:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
119 raise exceptions.DataError(_(u'invalid json option: {name}').format(name=option))
2436
b52e78cc86ed plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents: 2414
diff changeset
120 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None
b52e78cc86ed plugin import: fixed deserialisation of pubsub service
Goffi <goffi@goffi.org>
parents: 2414
diff changeset
121 return self.doImport(client, import_handler, unicode(name), unicode(location), options, pubsub_service, pubsub_node or None)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
122
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
123 @defer.inlineCallbacks
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
124 def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None, pubsub_node=None):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
125 """Import data
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
126
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
127 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
128 @param name(unicode): name of the importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
129 @param location(unicode): location of the data to import
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
130 can be an url, a file path, or anything which make sense
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
131 check importer description for more details
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
132 @param options(dict, None): extra options.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
133 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
134 None to use profile's server
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
135 @param pubsub_node(unicode, None): PubSub node to use
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
136 None to use importer's default node
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
137 @return (unicode): progress id
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
138 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
139 if options is None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
140 options = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
141 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
142 for opt_name, opt_default in import_handler.OPT_DEFAULTS.iteritems():
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
143 # we want a filled options dict, with all empty or False values removed
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
144 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
145 value =options[opt_name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
146 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
147 if opt_default:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
148 options[opt_name] = opt_default
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
149 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
150 if not value:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
151 del options[opt_name]
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
152
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
153 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
154 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
155 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
156 raise exceptions.NotFound(u"Importer [{}] not found".format(name))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
157 items_import_data, items_count = yield importer.callback(client, location, options)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
158 progress_id = unicode(uuid.uuid4())
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
159 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
160 _import = client._import
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
161 except AttributeError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
162 _import = client._import = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
163 progress_data = _import.setdefault(import_handler.name, {})
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
164 progress_data[progress_id] = {u'position': '0'}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
165 if items_count is not None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
166 progress_data[progress_id]['size'] = unicode(items_count)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
167 metadata = {'name': u'{}: {}'.format(name, location),
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
168 'direction': 'out',
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
169 'type': import_handler.name.upper() + '_IMPORT'
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
170 }
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
171 self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
172 self.host.bridge.progressStarted(progress_id, metadata, client.profile)
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
173 session = { # session data, can be used by importers
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
174 u'root_service': pubsub_service,
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
175 u'root_node': pubsub_node
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
176 }
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
177 self.recursiveImport(client, import_handler, items_import_data, progress_id, session, options, None, pubsub_service, pubsub_node)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
178 defer.returnValue(progress_id)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
179
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
180 @defer.inlineCallbacks
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
181 def recursiveImport(self, client, import_handler, items_import_data, progress_id, session, options, return_data=None, service=None, node=None, depth=0):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
182 """Do the import recursively
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
183
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
184 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
185 @param items_import_data(iterable): iterable of data as specified in [register]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
186 @param progress_id(unicode): id of progression
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
187 @param session(dict): data for this import session
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
188 can be used by importer so store any useful data
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
189 "root_service" and "root_node" are set to the main pubsub service and node of the import
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
190 @param options(dict): import options
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
191 @param return_data(dict): data to return on progressFinished
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
192 @param service(jid.JID, None): PubSub service to use
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
193 @param node(unicode, None): PubSub node to use
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
194 @param depth(int): level of recursion
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
195 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
196 if return_data is None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
197 return_data = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
198 for idx, item_import_data in enumerate(items_import_data):
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
199 item_data = yield import_handler.importItem(client, item_import_data, session, options, return_data, service, node)
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
200 yield import_handler.itemFilters(client, item_data, session, options)
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
201 recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, session, options)
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
202 yield import_handler.publishItem(client, item_data, service, node, session)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
203
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
204 if recurse_kwargs is not None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
205 recurse_kwargs['client'] = client
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
206 recurse_kwargs['import_handler'] = import_handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
207 recurse_kwargs['progress_id'] = progress_id
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
208 recurse_kwargs['session'] = session
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
209 recurse_kwargs.setdefault('options', options)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
210 recurse_kwargs['return_data'] = return_data
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
211 recurse_kwargs['depth'] = depth + 1
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
212 log.debug(_(u"uploading subitems"))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
213 yield self.recursiveImport(**recurse_kwargs)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
214
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
215 if depth == 0:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
216 client._import[import_handler.name][progress_id]['position'] = unicode(idx+1)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
217
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
218 if depth == 0:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
219 self.host.bridge.progressFinished(progress_id,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
220 return_data,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
221 client.profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
222 self.host.removeProgressCb(progress_id, client.profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
223 del client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
224
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
225 def register(self, import_handler, name, callback, short_desc='', long_desc=''):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
226 """Register an Importer method
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
227
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
228 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
229 @param callback(callable): method to call:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
230 the signature must be (client, location, options) (cf. [doImport])
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
231 the importer must return a tuple with (items_import_data, items_count)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
232 items_import_data(iterable[dict]) data specific to specialized importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
233 cf. importItem docstring of specialized importer for details
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
234 items_count (int, None) indicate the total number of items (without subitems)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
235 useful to display a progress indicator when the iterator is a generator
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
236 use None if you can't guess the total number of items
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
237 @param short_desc(unicode): one line description of the importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
238 @param long_desc(unicode): long description of the importer, its options, etc.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
239 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
240 name = name.lower()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
241 if name in import_handler.importers:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
242 raise exceptions.ConflictError(_(u"An {handler_name} importer with the name {name} already exist").format(
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
243 handler_name = import_handler.name,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
244 name = name))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
245 import_handler.importers[name] = Importer(callback, short_desc, long_desc)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
246
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
247 def unregister(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
248 del import_handler.importers[name]