Mercurial > libervia-backend
comparison sat/plugins/plugin_import.py @ 2624:56f94936df1e
code style reformatting using black
author | Goffi <goffi@goffi.org> |
---|---|
date | Wed, 27 Jun 2018 20:14:46 +0200 |
parents | 26edcf3a30eb |
children | 003b8b4b56a7 |
comparison
equal
deleted
inserted
replaced
2623:49533de4540b | 2624:56f94936df1e |
---|---|
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. | 18 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
19 | 19 |
20 from sat.core.i18n import _ | 20 from sat.core.i18n import _ |
21 from sat.core.constants import Const as C | 21 from sat.core.constants import Const as C |
22 from sat.core.log import getLogger | 22 from sat.core.log import getLogger |
23 | |
23 log = getLogger(__name__) | 24 log = getLogger(__name__) |
24 from twisted.internet import defer | 25 from twisted.internet import defer |
25 from sat.core import exceptions | 26 from sat.core import exceptions |
26 from twisted.words.protocols.jabber import jid | 27 from twisted.words.protocols.jabber import jid |
27 from functools import partial | 28 from functools import partial |
35 C.PI_IMPORT_NAME: "IMPORT", | 36 C.PI_IMPORT_NAME: "IMPORT", |
36 C.PI_TYPE: C.PLUG_TYPE_IMPORT, | 37 C.PI_TYPE: C.PLUG_TYPE_IMPORT, |
37 C.PI_DEPENDENCIES: [], | 38 C.PI_DEPENDENCIES: [], |
38 C.PI_MAIN: "ImportPlugin", | 39 C.PI_MAIN: "ImportPlugin", |
39 C.PI_HANDLER: "no", | 40 C.PI_HANDLER: "no", |
40 C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers""") | 41 C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers"""), |
41 } | 42 } |
42 | 43 |
43 Importer = collections.namedtuple('Importer', ('callback', 'short_desc', 'long_desc')) | 44 Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc")) |
44 | 45 |
45 | 46 |
46 class ImportPlugin(object): | 47 class ImportPlugin(object): |
47 | |
48 def __init__(self, host): | 48 def __init__(self, host): |
49 log.info(_("plugin Import initialization")) | 49 log.info(_("plugin Import initialization")) |
50 self.host = host | 50 self.host = host |
51 | 51 |
52 def initialize(self, import_handler, name): | 52 def initialize(self, import_handler, name): |
62 - publishItem: actualy publish an item | 62 - publishItem: actualy publish an item |
63 - itemFilters: modify item according to options | 63 - itemFilters: modify item according to options |
64 @param name(unicode): import handler name | 64 @param name(unicode): import handler name |
65 """ | 65 """ |
66 assert name == name.lower().strip() | 66 assert name == name.lower().strip() |
67 log.info(_(u'initializing {name} import handler').format(name=name)) | 67 log.info(_(u"initializing {name} import handler").format(name=name)) |
68 import_handler.name = name | 68 import_handler.name = name |
69 import_handler.register = partial(self.register, import_handler) | 69 import_handler.register = partial(self.register, import_handler) |
70 import_handler.unregister = partial(self.unregister, import_handler) | 70 import_handler.unregister = partial(self.unregister, import_handler) |
71 import_handler.importers = {} | 71 import_handler.importers = {} |
72 | |
72 def _import(name, location, options, pubsub_service, pubsub_node, profile): | 73 def _import(name, location, options, pubsub_service, pubsub_node, profile): |
73 return self._doImport(import_handler, name, location, options, pubsub_service, pubsub_node, profile) | 74 return self._doImport( |
75 import_handler, | |
76 name, | |
77 location, | |
78 options, | |
79 pubsub_service, | |
80 pubsub_node, | |
81 profile, | |
82 ) | |
83 | |
74 def _importList(): | 84 def _importList(): |
75 return self.listImporters(import_handler) | 85 return self.listImporters(import_handler) |
86 | |
76 def _importDesc(name): | 87 def _importDesc(name): |
77 return self.getDescription(import_handler, name) | 88 return self.getDescription(import_handler, name) |
78 | 89 |
79 self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}sss', out_sign='s', method=_import, async=True) | 90 self.host.bridge.addMethod( |
80 self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList) | 91 name + "Import", |
81 self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc) | 92 ".plugin", |
93 in_sign="ssa{ss}sss", | |
94 out_sign="s", | |
95 method=_import, | |
96 async=True, | |
97 ) | |
98 self.host.bridge.addMethod( | |
99 name + "ImportList", | |
100 ".plugin", | |
101 in_sign="", | |
102 out_sign="a(ss)", | |
103 method=_importList, | |
104 ) | |
105 self.host.bridge.addMethod( | |
106 name + "ImportDesc", | |
107 ".plugin", | |
108 in_sign="s", | |
109 out_sign="(ss)", | |
110 method=_importDesc, | |
111 ) | |
82 | 112 |
83 def getProgress(self, import_handler, progress_id, profile): | 113 def getProgress(self, import_handler, progress_id, profile): |
84 client = self.host.getClient(profile) | 114 client = self.host.getClient(profile) |
85 return client._import[import_handler.name][progress_id] | 115 return client._import[import_handler.name][progress_id] |
86 | 116 |
87 def listImporters(self, import_handler): | 117 def listImporters(self, import_handler): |
88 importers = import_handler.importers.keys() | 118 importers = import_handler.importers.keys() |
89 importers.sort() | 119 importers.sort() |
90 return [(name, import_handler.importers[name].short_desc) for name in import_handler.importers] | 120 return [ |
121 (name, import_handler.importers[name].short_desc) | |
122 for name in import_handler.importers | |
123 ] | |
91 | 124 |
92 def getDescription(self, import_handler, name): | 125 def getDescription(self, import_handler, name): |
93 """Return import short and long descriptions | 126 """Return import short and long descriptions |
94 | 127 |
95 @param name(unicode): importer name | 128 @param name(unicode): importer name |
96 @return (tuple[unicode,unicode]): short and long description | 129 @return (tuple[unicode,unicode]): short and long description |
97 """ | 130 """ |
98 try: | 131 try: |
99 importer = import_handler.importers[name] | 132 importer = import_handler.importers[name] |
100 except KeyError: | 133 except KeyError: |
101 raise exceptions.NotFound(u"{handler_name} importer not found [{name}]".format( | 134 raise exceptions.NotFound( |
102 handler_name = import_handler.name, | 135 u"{handler_name} importer not found [{name}]".format( |
103 name = name)) | 136 handler_name=import_handler.name, name=name |
137 ) | |
138 ) | |
104 else: | 139 else: |
105 return importer.short_desc, importer.long_desc | 140 return importer.short_desc, importer.long_desc |
106 | 141 |
107 def _doImport(self, import_handler, name, location, options, pubsub_service='', pubsub_node='', profile=C.PROF_KEY_NONE): | 142 def _doImport( |
143 self, | |
144 import_handler, | |
145 name, | |
146 location, | |
147 options, | |
148 pubsub_service="", | |
149 pubsub_node="", | |
150 profile=C.PROF_KEY_NONE, | |
151 ): | |
108 client = self.host.getClient(profile) | 152 client = self.host.getClient(profile) |
109 options = {key: unicode(value) for key, value in options.iteritems()} | 153 options = {key: unicode(value) for key, value in options.iteritems()} |
110 for option in import_handler.BOOL_OPTIONS: | 154 for option in import_handler.BOOL_OPTIONS: |
111 try: | 155 try: |
112 options[option] = C.bool(options[option]) | 156 options[option] = C.bool(options[option]) |
114 pass | 158 pass |
115 for option in import_handler.JSON_OPTIONS: | 159 for option in import_handler.JSON_OPTIONS: |
116 try: | 160 try: |
117 options[option] = json.loads(options[option]) | 161 options[option] = json.loads(options[option]) |
118 except ValueError: | 162 except ValueError: |
119 raise exceptions.DataError(_(u'invalid json option: {name}').format(name=option)) | 163 raise exceptions.DataError( |
164 _(u"invalid json option: {name}").format(name=option) | |
165 ) | |
120 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None | 166 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None |
121 return self.doImport(client, import_handler, unicode(name), unicode(location), options, pubsub_service, pubsub_node or None) | 167 return self.doImport( |
168 client, | |
169 import_handler, | |
170 unicode(name), | |
171 unicode(location), | |
172 options, | |
173 pubsub_service, | |
174 pubsub_node or None, | |
175 ) | |
122 | 176 |
123 @defer.inlineCallbacks | 177 @defer.inlineCallbacks |
124 def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None, pubsub_node=None): | 178 def doImport( |
179 self, | |
180 client, | |
181 import_handler, | |
182 name, | |
183 location, | |
184 options=None, | |
185 pubsub_service=None, | |
186 pubsub_node=None, | |
187 ): | |
125 """Import data | 188 """Import data |
126 | 189 |
127 @param import_handler(object): instance of the import handler | 190 @param import_handler(object): instance of the import handler |
128 @param name(unicode): name of the importer | 191 @param name(unicode): name of the importer |
129 @param location(unicode): location of the data to import | 192 @param location(unicode): location of the data to import |
140 options = {} | 203 options = {} |
141 else: | 204 else: |
142 for opt_name, opt_default in import_handler.OPT_DEFAULTS.iteritems(): | 205 for opt_name, opt_default in import_handler.OPT_DEFAULTS.iteritems(): |
143 # we want a filled options dict, with all empty or False values removed | 206 # we want a filled options dict, with all empty or False values removed |
144 try: | 207 try: |
145 value =options[opt_name] | 208 value = options[opt_name] |
146 except KeyError: | 209 except KeyError: |
147 if opt_default: | 210 if opt_default: |
148 options[opt_name] = opt_default | 211 options[opt_name] = opt_default |
149 else: | 212 else: |
150 if not value: | 213 if not value: |
152 | 215 |
153 try: | 216 try: |
154 importer = import_handler.importers[name] | 217 importer = import_handler.importers[name] |
155 except KeyError: | 218 except KeyError: |
156 raise exceptions.NotFound(u"Importer [{}] not found".format(name)) | 219 raise exceptions.NotFound(u"Importer [{}] not found".format(name)) |
157 items_import_data, items_count = yield importer.callback(client, location, options) | 220 items_import_data, items_count = yield importer.callback( |
221 client, location, options | |
222 ) | |
158 progress_id = unicode(uuid.uuid4()) | 223 progress_id = unicode(uuid.uuid4()) |
159 try: | 224 try: |
160 _import = client._import | 225 _import = client._import |
161 except AttributeError: | 226 except AttributeError: |
162 _import = client._import = {} | 227 _import = client._import = {} |
163 progress_data = _import.setdefault(import_handler.name, {}) | 228 progress_data = _import.setdefault(import_handler.name, {}) |
164 progress_data[progress_id] = {u'position': '0'} | 229 progress_data[progress_id] = {u"position": "0"} |
165 if items_count is not None: | 230 if items_count is not None: |
166 progress_data[progress_id]['size'] = unicode(items_count) | 231 progress_data[progress_id]["size"] = unicode(items_count) |
167 metadata = {'name': u'{}: {}'.format(name, location), | 232 metadata = { |
168 'direction': 'out', | 233 "name": u"{}: {}".format(name, location), |
169 'type': import_handler.name.upper() + '_IMPORT' | 234 "direction": "out", |
170 } | 235 "type": import_handler.name.upper() + "_IMPORT", |
171 self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile) | 236 } |
237 self.host.registerProgressCb( | |
238 progress_id, | |
239 partial(self.getProgress, import_handler), | |
240 metadata, | |
241 profile=client.profile, | |
242 ) | |
172 self.host.bridge.progressStarted(progress_id, metadata, client.profile) | 243 self.host.bridge.progressStarted(progress_id, metadata, client.profile) |
173 session = { # session data, can be used by importers | 244 session = { # session data, can be used by importers |
174 u'root_service': pubsub_service, | 245 u"root_service": pubsub_service, |
175 u'root_node': pubsub_node | 246 u"root_node": pubsub_node, |
176 } | 247 } |
177 self.recursiveImport(client, import_handler, items_import_data, progress_id, session, options, None, pubsub_service, pubsub_node) | 248 self.recursiveImport( |
249 client, | |
250 import_handler, | |
251 items_import_data, | |
252 progress_id, | |
253 session, | |
254 options, | |
255 None, | |
256 pubsub_service, | |
257 pubsub_node, | |
258 ) | |
178 defer.returnValue(progress_id) | 259 defer.returnValue(progress_id) |
179 | 260 |
180 @defer.inlineCallbacks | 261 @defer.inlineCallbacks |
181 def recursiveImport(self, client, import_handler, items_import_data, progress_id, session, options, return_data=None, service=None, node=None, depth=0): | 262 def recursiveImport( |
263 self, | |
264 client, | |
265 import_handler, | |
266 items_import_data, | |
267 progress_id, | |
268 session, | |
269 options, | |
270 return_data=None, | |
271 service=None, | |
272 node=None, | |
273 depth=0, | |
274 ): | |
182 """Do the import recursively | 275 """Do the import recursively |
183 | 276 |
184 @param import_handler(object): instance of the import handler | 277 @param import_handler(object): instance of the import handler |
185 @param items_import_data(iterable): iterable of data as specified in [register] | 278 @param items_import_data(iterable): iterable of data as specified in [register] |
186 @param progress_id(unicode): id of progression | 279 @param progress_id(unicode): id of progression |
194 @param depth(int): level of recursion | 287 @param depth(int): level of recursion |
195 """ | 288 """ |
196 if return_data is None: | 289 if return_data is None: |
197 return_data = {} | 290 return_data = {} |
198 for idx, item_import_data in enumerate(items_import_data): | 291 for idx, item_import_data in enumerate(items_import_data): |
199 item_data = yield import_handler.importItem(client, item_import_data, session, options, return_data, service, node) | 292 item_data = yield import_handler.importItem( |
293 client, item_import_data, session, options, return_data, service, node | |
294 ) | |
200 yield import_handler.itemFilters(client, item_data, session, options) | 295 yield import_handler.itemFilters(client, item_data, session, options) |
201 recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, session, options) | 296 recurse_kwargs = yield import_handler.importSubItems( |
297 client, item_import_data, item_data, session, options | |
298 ) | |
202 yield import_handler.publishItem(client, item_data, service, node, session) | 299 yield import_handler.publishItem(client, item_data, service, node, session) |
203 | 300 |
204 if recurse_kwargs is not None: | 301 if recurse_kwargs is not None: |
205 recurse_kwargs['client'] = client | 302 recurse_kwargs["client"] = client |
206 recurse_kwargs['import_handler'] = import_handler | 303 recurse_kwargs["import_handler"] = import_handler |
207 recurse_kwargs['progress_id'] = progress_id | 304 recurse_kwargs["progress_id"] = progress_id |
208 recurse_kwargs['session'] = session | 305 recurse_kwargs["session"] = session |
209 recurse_kwargs.setdefault('options', options) | 306 recurse_kwargs.setdefault("options", options) |
210 recurse_kwargs['return_data'] = return_data | 307 recurse_kwargs["return_data"] = return_data |
211 recurse_kwargs['depth'] = depth + 1 | 308 recurse_kwargs["depth"] = depth + 1 |
212 log.debug(_(u"uploading subitems")) | 309 log.debug(_(u"uploading subitems")) |
213 yield self.recursiveImport(**recurse_kwargs) | 310 yield self.recursiveImport(**recurse_kwargs) |
214 | 311 |
215 if depth == 0: | 312 if depth == 0: |
216 client._import[import_handler.name][progress_id]['position'] = unicode(idx+1) | 313 client._import[import_handler.name][progress_id]["position"] = unicode( |
314 idx + 1 | |
315 ) | |
217 | 316 |
218 if depth == 0: | 317 if depth == 0: |
219 self.host.bridge.progressFinished(progress_id, | 318 self.host.bridge.progressFinished(progress_id, return_data, client.profile) |
220 return_data, | |
221 client.profile) | |
222 self.host.removeProgressCb(progress_id, client.profile) | 319 self.host.removeProgressCb(progress_id, client.profile) |
223 del client._import[import_handler.name][progress_id] | 320 del client._import[import_handler.name][progress_id] |
224 | 321 |
225 def register(self, import_handler, name, callback, short_desc='', long_desc=''): | 322 def register(self, import_handler, name, callback, short_desc="", long_desc=""): |
226 """Register an Importer method | 323 """Register an Importer method |
227 | 324 |
228 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase | 325 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase |
229 @param callback(callable): method to call: | 326 @param callback(callable): method to call: |
230 the signature must be (client, location, options) (cf. [doImport]) | 327 the signature must be (client, location, options) (cf. [doImport]) |
237 @param short_desc(unicode): one line description of the importer | 334 @param short_desc(unicode): one line description of the importer |
238 @param long_desc(unicode): long description of the importer, its options, etc. | 335 @param long_desc(unicode): long description of the importer, its options, etc. |
239 """ | 336 """ |
240 name = name.lower() | 337 name = name.lower() |
241 if name in import_handler.importers: | 338 if name in import_handler.importers: |
242 raise exceptions.ConflictError(_(u"An {handler_name} importer with the name {name} already exist").format( | 339 raise exceptions.ConflictError( |
243 handler_name = import_handler.name, | 340 _( |
244 name = name)) | 341 u"An {handler_name} importer with the name {name} already exist" |
342 ).format(handler_name=import_handler.name, name=name) | |
343 ) | |
245 import_handler.importers[name] = Importer(callback, short_desc, long_desc) | 344 import_handler.importers[name] = Importer(callback, short_desc, long_desc) |
246 | 345 |
247 def unregister(self, import_handler, name): | 346 def unregister(self, import_handler, name): |
248 del import_handler.importers[name] | 347 del import_handler.importers[name] |