Mercurial > libervia-backend

annotate src/plugins/plugin_import.py @ 2403:dec31114c402

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
template: improved date formatter: the date filter is now named "date_fmt" and can be use to specify several useful formats (see docstring), using the current locale. The "relative" format return the whole human readable relative date. The new in_the_past test allow to check if a date is passed.
author Goffi <goffi@goffi.org>
date Fri, 27 Oct 2017 18:20:10 +0200
parents 66baa687c682
children 8b37a62336c3
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
1 #!/usr/bin/env python2
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
2 # -*- coding: utf-8 -*-
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
3
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
4 # SàT plugin for generic data import handling
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
5 # Copyright (C) 2009-2016 Jérôme Poisson (goffi@goffi.org)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
6
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
7 # This program is free software: you can redistribute it and/or modify
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
8 # it under the terms of the GNU Affero General Public License as published by
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
9 # the Free Software Foundation, either version 3 of the License, or
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
10 # (at your option) any later version.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
11
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
12 # This program is distributed in the hope that it will be useful,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
15 # GNU Affero General Public License for more details.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
16
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
17 # You should have received a copy of the GNU Affero General Public License
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
19
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
20 from sat.core.i18n import _
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
21 from sat.core.constants import Const as C
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
22 from sat.core.log import getLogger
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
23 log = getLogger(__name__)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
24 from twisted.internet import defer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
25 from sat.core import exceptions
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
26 from functools import partial
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
27 import collections
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
28 import uuid
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
29 import json
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
30
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
31
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
32 PLUGIN_INFO = {
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
33 C.PI_NAME: "import",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
34 C.PI_IMPORT_NAME: "IMPORT",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
35 C.PI_TYPE: C.PLUG_TYPE_IMPORT,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
36 C.PI_DEPENDENCIES: [],
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
37 C.PI_MAIN: "ImportPlugin",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
38 C.PI_HANDLER: "no",
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
39 C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers""")
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
40 }
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
41
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
42 Importer = collections.namedtuple('Importer', ('callback', 'short_desc', 'long_desc'))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
43
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
44
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
45 class ImportPlugin(object):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
46
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
47 def __init__(self, host):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
48 log.info(_("plugin Import initialization"))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
49 self.host = host
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
50
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
51 def initialize(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
52 """Initialize a specialized import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
53
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
54 @param import_handler(object): specialized import handler instance
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
55 must have the following methods:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
56 - importItem: import a single main item (i.e. prepare data for publishing)
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
57 - importSubitems: import sub items (i.e. items linked to main item, e.g. comments).
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
58 Must return a dict with kwargs for recursiveImport if items are to be imported recursively.
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
59 At least "items_import_data", "service" and "node" keys must be provided.
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
60 if None is returned, no recursion will be done to import subitems, but import can still be done directly by the method.
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
61 - publishItem: actualy publish an item
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
62 - itemFilters: modify item according to options
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
63 @param name(unicode): import handler name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
64 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
65 assert name == name.lower().strip()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
66 log.info(_(u'initializing {name} import handler').format(name=name))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
67 import_handler.name = name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
68 import_handler.register = partial(self.register, import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
69 import_handler.unregister = partial(self.unregister, import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
70 import_handler.importers = {}
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
71 def _import(name, location, options, pubsub_service, pubsub_node, profile):
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
72 return self._doImport(import_handler, name, location, options, pubsub_service, pubsub_node, profile)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
73 def _importList():
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
74 return self.listImporters(import_handler)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
75 def _importDesc(name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
76 return self.getDescription(import_handler, name)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
77
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
78 self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}sss', out_sign='s', method=_import, async=True)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
79 self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
80 self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
81
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
82 def getProgress(self, import_handler, progress_id, profile):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
83 client = self.host.getClient(profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
84 return client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
85
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
86 def listImporters(self, import_handler):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
87 importers = import_handler.importers.keys()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
88 importers.sort()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
89 return [(name, import_handler.importers[name].short_desc) for name in import_handler.importers]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
90
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
91 def getDescription(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
92 """Return import short and long descriptions
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
93
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
94 @param name(unicode): importer name
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
95 @return (tuple[unicode,unicode]): short and long description
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
96 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
97 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
98 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
99 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
100 raise exceptions.NotFound(u"{handler_name} importer not found [{name}]".format(
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
101 handler_name = import_handler.name,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
102 name = name))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
103 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
104 return importer.short_desc, importer.long_desc
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
105
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
106 def _doImport(self, import_handler, name, location, options, pubsub_service='', pubsub_node='', profile=C.PROF_KEY_NONE):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
107 client = self.host.getClient(profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
108 options = {key: unicode(value) for key, value in options.iteritems()}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
109 for option in import_handler.BOOL_OPTIONS:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
110 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
111 options[option] = C.bool(options[option])
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
112 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
113 pass
2396
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
114 for option in import_handler.JSON_OPTIONS:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
115 try:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
116 options[option] = json.loads(options[option])
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
117 except ValueError:
66baa687c682 plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents: 2390
diff changeset
118 raise exceptions.DataError(_(u'invalid json option: {name}').format(name=option))
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
119 return self.doImport(client, import_handler, unicode(name), unicode(location), options, pubsub_service or None, pubsub_node or None)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
120
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
121 @defer.inlineCallbacks
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
122 def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None, pubsub_node=None):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
123 """Import data
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
124
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
125 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
126 @param name(unicode): name of the importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
127 @param location(unicode): location of the data to import
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
128 can be an url, a file path, or anything which make sense
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
129 check importer description for more details
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
130 @param options(dict, None): extra options.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
131 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
132 None to use profile's server
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
133 @param pubsub_node(unicode, None): PubSub node to use
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
134 None to use importer's default node
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
135 @return (unicode): progress id
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
136 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
137 if options is None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
138 options = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
139 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
140 for opt_name, opt_default in import_handler.OPT_DEFAULTS.iteritems():
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
141 # we want a filled options dict, with all empty or False values removed
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
142 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
143 value =options[opt_name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
144 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
145 if opt_default:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
146 options[opt_name] = opt_default
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
147 else:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
148 if not value:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
149 del options[opt_name]
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
150
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
151 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
152 importer = import_handler.importers[name]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
153 except KeyError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
154 raise exceptions.NotFound(u"Importer [{}] not found".format(name))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
155 items_import_data, items_count = yield importer.callback(client, location, options)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
156 progress_id = unicode(uuid.uuid4())
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
157 try:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
158 _import = client._import
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
159 except AttributeError:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
160 _import = client._import = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
161 progress_data = _import.setdefault(import_handler.name, {})
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
162 progress_data[progress_id] = {u'position': '0'}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
163 if items_count is not None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
164 progress_data[progress_id]['size'] = unicode(items_count)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
165 metadata = {'name': u'{}: {}'.format(name, location),
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
166 'direction': 'out',
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
167 'type': import_handler.name.upper() + '_IMPORT'
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
168 }
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
169 self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
170 self.host.bridge.progressStarted(progress_id, metadata, client.profile)
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
171 session = { # session data, can be used by importers
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
172 u'root_service': pubsub_service,
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
173 u'root_node': pubsub_node
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
174 }
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
175 self.recursiveImport(client, import_handler, items_import_data, progress_id, session, options, None, pubsub_service, pubsub_node)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
176 defer.returnValue(progress_id)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
177
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
178 @defer.inlineCallbacks
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
179 def recursiveImport(self, client, import_handler, items_import_data, progress_id, session, options, return_data=None, service=None, node=None, depth=0):
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
180 """Do the import recursively
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
181
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
182 @param import_handler(object): instance of the import handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
183 @param items_import_data(iterable): iterable of data as specified in [register]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
184 @param progress_id(unicode): id of progression
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
185 @param session(dict): data for this import session
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
186 can be used by importer so store any useful data
2390
f57a8eaec8ed plugins import, tickets import, bugzilla import: comments handling:
Goffi <goffi@goffi.org>
parents: 2370
diff changeset
187 "root_service" and "root_node" are set to the main pubsub service and node of the import
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
188 @param options(dict): import options
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
189 @param return_data(dict): data to return on progressFinished
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
190 @param service(jid.JID, None): PubSub service to use
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
191 @param node(unicode, None): PubSub node to use
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
192 @param depth(int): level of recursion
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
193 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
194 if return_data is None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
195 return_data = {}
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
196 for idx, item_import_data in enumerate(items_import_data):
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
197 item_data = yield import_handler.importItem(client, item_import_data, session, options, return_data, service, node)
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
198 yield import_handler.itemFilters(client, item_data, session, options)
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
199 recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, session, options)
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
200 yield import_handler.publishItem(client, item_data, service, node, session)
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
201
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
202 if recurse_kwargs is not None:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
203 recurse_kwargs['client'] = client
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
204 recurse_kwargs['import_handler'] = import_handler
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
205 recurse_kwargs['progress_id'] = progress_id
2370
2c2b826b0bb3 plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents: 2369
diff changeset
206 recurse_kwargs['session'] = session
2369
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
207 recurse_kwargs.setdefault('options', options)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
208 recurse_kwargs['return_data'] = return_data
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
209 recurse_kwargs['depth'] = depth + 1
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
210 log.debug(_(u"uploading subitems"))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
211 yield self.recursiveImport(**recurse_kwargs)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
212
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
213 if depth == 0:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
214 client._import[import_handler.name][progress_id]['position'] = unicode(idx+1)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
215
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
216 if depth == 0:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
217 self.host.bridge.progressFinished(progress_id,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
218 return_data,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
219 client.profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
220 self.host.removeProgressCb(progress_id, client.profile)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
221 del client._import[import_handler.name][progress_id]
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
222
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
223 def register(self, import_handler, name, callback, short_desc='', long_desc=''):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
224 """Register an Importer method
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
225
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
226 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
227 @param callback(callable): method to call:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
228 the signature must be (client, location, options) (cf. [doImport])
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
229 the importer must return a tuple with (items_import_data, items_count)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
230 items_import_data(iterable[dict]) data specific to specialized importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
231 cf. importItem docstring of specialized importer for details
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
232 items_count (int, None) indicate the total number of items (without subitems)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
233 useful to display a progress indicator when the iterator is a generator
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
234 use None if you can't guess the total number of items
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
235 @param short_desc(unicode): one line description of the importer
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
236 @param long_desc(unicode): long description of the importer, its options, etc.
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
237 """
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
238 name = name.lower()
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
239 if name in import_handler.importers:
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
240 raise exceptions.ConflictError(_(u"An {handler_name} importer with the name {name} already exist").format(
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
241 handler_name = import_handler.name,
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
242 name = name))
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
243 import_handler.importers[name] = Importer(callback, short_desc, long_desc)
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
244
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
245 def unregister(self, import_handler, name):
cdaa58e14553 plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
diff changeset
246 del import_handler.importers[name]