Mercurial > libervia-backend
comparison sat/plugins/plugin_import.py @ 3028:ab2696e34d29
Python 3 port:
/!\ this is a huge commit
/!\ starting from this commit, SàT is needs Python 3.6+
/!\ SàT maybe be instable or some feature may not work anymore, this will improve with time
This patch port backend, bridge and frontends to Python 3.
Roughly this has been done this way:
- 2to3 tools has been applied (with python 3.7)
- all references to python2 have been replaced with python3 (notably shebangs)
- fixed files not handled by 2to3 (notably the shell script)
- several manual fixes
- fixed issues reported by Python 3 that where not handled in Python 2
- replaced "async" with "async_" when needed (it's a reserved word from Python 3.7)
- replaced zope's "implements" with @implementer decorator
- temporary hack to handle data pickled in database, as str or bytes may be returned,
to be checked later
- fixed hash comparison for password
- removed some code which is not needed anymore with Python 3
- deactivated some code which needs to be checked (notably certificate validation)
- tested with jp, fixed reported issues until some basic commands worked
- ported Primitivus (after porting dependencies like urwid satext)
- more manual fixes
author | Goffi <goffi@goffi.org> |
---|---|
date | Tue, 13 Aug 2019 19:08:41 +0200 |
parents | 003b8b4b56a7 |
children | fee60f17ebac |
comparison
equal
deleted
inserted
replaced
3027:ff5bcb12ae60 | 3028:ab2696e34d29 |
---|---|
1 #!/usr/bin/env python2 | 1 #!/usr/bin/env python3 |
2 # -*- coding: utf-8 -*- | 2 # -*- coding: utf-8 -*- |
3 | 3 |
4 # SàT plugin for generic data import handling | 4 # SàT plugin for generic data import handling |
5 # Copyright (C) 2009-2019 Jérôme Poisson (goffi@goffi.org) | 5 # Copyright (C) 2009-2019 Jérôme Poisson (goffi@goffi.org) |
6 | 6 |
36 C.PI_IMPORT_NAME: "IMPORT", | 36 C.PI_IMPORT_NAME: "IMPORT", |
37 C.PI_TYPE: C.PLUG_TYPE_IMPORT, | 37 C.PI_TYPE: C.PLUG_TYPE_IMPORT, |
38 C.PI_DEPENDENCIES: [], | 38 C.PI_DEPENDENCIES: [], |
39 C.PI_MAIN: "ImportPlugin", | 39 C.PI_MAIN: "ImportPlugin", |
40 C.PI_HANDLER: "no", | 40 C.PI_HANDLER: "no", |
41 C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers"""), | 41 C.PI_DESCRIPTION: _("""Generic import plugin, base for specialized importers"""), |
42 } | 42 } |
43 | 43 |
44 Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc")) | 44 Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc")) |
45 | 45 |
46 | 46 |
62 - publishItem: actualy publish an item | 62 - publishItem: actualy publish an item |
63 - itemFilters: modify item according to options | 63 - itemFilters: modify item according to options |
64 @param name(unicode): import handler name | 64 @param name(unicode): import handler name |
65 """ | 65 """ |
66 assert name == name.lower().strip() | 66 assert name == name.lower().strip() |
67 log.info(_(u"initializing {name} import handler").format(name=name)) | 67 log.info(_("initializing {name} import handler").format(name=name)) |
68 import_handler.name = name | 68 import_handler.name = name |
69 import_handler.register = partial(self.register, import_handler) | 69 import_handler.register = partial(self.register, import_handler) |
70 import_handler.unregister = partial(self.unregister, import_handler) | 70 import_handler.unregister = partial(self.unregister, import_handler) |
71 import_handler.importers = {} | 71 import_handler.importers = {} |
72 | 72 |
91 name + "Import", | 91 name + "Import", |
92 ".plugin", | 92 ".plugin", |
93 in_sign="ssa{ss}sss", | 93 in_sign="ssa{ss}sss", |
94 out_sign="s", | 94 out_sign="s", |
95 method=_import, | 95 method=_import, |
96 async=True, | 96 async_=True, |
97 ) | 97 ) |
98 self.host.bridge.addMethod( | 98 self.host.bridge.addMethod( |
99 name + "ImportList", | 99 name + "ImportList", |
100 ".plugin", | 100 ".plugin", |
101 in_sign="", | 101 in_sign="", |
113 def getProgress(self, import_handler, progress_id, profile): | 113 def getProgress(self, import_handler, progress_id, profile): |
114 client = self.host.getClient(profile) | 114 client = self.host.getClient(profile) |
115 return client._import[import_handler.name][progress_id] | 115 return client._import[import_handler.name][progress_id] |
116 | 116 |
117 def listImporters(self, import_handler): | 117 def listImporters(self, import_handler): |
118 importers = import_handler.importers.keys() | 118 importers = list(import_handler.importers.keys()) |
119 importers.sort() | 119 importers.sort() |
120 return [ | 120 return [ |
121 (name, import_handler.importers[name].short_desc) | 121 (name, import_handler.importers[name].short_desc) |
122 for name in import_handler.importers | 122 for name in import_handler.importers |
123 ] | 123 ] |
130 """ | 130 """ |
131 try: | 131 try: |
132 importer = import_handler.importers[name] | 132 importer = import_handler.importers[name] |
133 except KeyError: | 133 except KeyError: |
134 raise exceptions.NotFound( | 134 raise exceptions.NotFound( |
135 u"{handler_name} importer not found [{name}]".format( | 135 "{handler_name} importer not found [{name}]".format( |
136 handler_name=import_handler.name, name=name | 136 handler_name=import_handler.name, name=name |
137 ) | 137 ) |
138 ) | 138 ) |
139 else: | 139 else: |
140 return importer.short_desc, importer.long_desc | 140 return importer.short_desc, importer.long_desc |
148 pubsub_service="", | 148 pubsub_service="", |
149 pubsub_node="", | 149 pubsub_node="", |
150 profile=C.PROF_KEY_NONE, | 150 profile=C.PROF_KEY_NONE, |
151 ): | 151 ): |
152 client = self.host.getClient(profile) | 152 client = self.host.getClient(profile) |
153 options = {key: unicode(value) for key, value in options.iteritems()} | 153 options = {key: str(value) for key, value in options.items()} |
154 for option in import_handler.BOOL_OPTIONS: | 154 for option in import_handler.BOOL_OPTIONS: |
155 try: | 155 try: |
156 options[option] = C.bool(options[option]) | 156 options[option] = C.bool(options[option]) |
157 except KeyError: | 157 except KeyError: |
158 pass | 158 pass |
159 for option in import_handler.JSON_OPTIONS: | 159 for option in import_handler.JSON_OPTIONS: |
160 try: | 160 try: |
161 options[option] = json.loads(options[option]) | 161 options[option] = json.loads(options[option]) |
162 except ValueError: | 162 except ValueError: |
163 raise exceptions.DataError( | 163 raise exceptions.DataError( |
164 _(u"invalid json option: {name}").format(name=option) | 164 _("invalid json option: {name}").format(name=option) |
165 ) | 165 ) |
166 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None | 166 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None |
167 return self.doImport( | 167 return self.doImport( |
168 client, | 168 client, |
169 import_handler, | 169 import_handler, |
170 unicode(name), | 170 str(name), |
171 unicode(location), | 171 str(location), |
172 options, | 172 options, |
173 pubsub_service, | 173 pubsub_service, |
174 pubsub_node or None, | 174 pubsub_node or None, |
175 ) | 175 ) |
176 | 176 |
200 @return (unicode): progress id | 200 @return (unicode): progress id |
201 """ | 201 """ |
202 if options is None: | 202 if options is None: |
203 options = {} | 203 options = {} |
204 else: | 204 else: |
205 for opt_name, opt_default in import_handler.OPT_DEFAULTS.iteritems(): | 205 for opt_name, opt_default in import_handler.OPT_DEFAULTS.items(): |
206 # we want a filled options dict, with all empty or False values removed | 206 # we want a filled options dict, with all empty or False values removed |
207 try: | 207 try: |
208 value = options[opt_name] | 208 value = options[opt_name] |
209 except KeyError: | 209 except KeyError: |
210 if opt_default: | 210 if opt_default: |
214 del options[opt_name] | 214 del options[opt_name] |
215 | 215 |
216 try: | 216 try: |
217 importer = import_handler.importers[name] | 217 importer = import_handler.importers[name] |
218 except KeyError: | 218 except KeyError: |
219 raise exceptions.NotFound(u"Importer [{}] not found".format(name)) | 219 raise exceptions.NotFound("Importer [{}] not found".format(name)) |
220 items_import_data, items_count = yield importer.callback( | 220 items_import_data, items_count = yield importer.callback( |
221 client, location, options | 221 client, location, options |
222 ) | 222 ) |
223 progress_id = unicode(uuid.uuid4()) | 223 progress_id = str(uuid.uuid4()) |
224 try: | 224 try: |
225 _import = client._import | 225 _import = client._import |
226 except AttributeError: | 226 except AttributeError: |
227 _import = client._import = {} | 227 _import = client._import = {} |
228 progress_data = _import.setdefault(import_handler.name, {}) | 228 progress_data = _import.setdefault(import_handler.name, {}) |
229 progress_data[progress_id] = {u"position": "0"} | 229 progress_data[progress_id] = {"position": "0"} |
230 if items_count is not None: | 230 if items_count is not None: |
231 progress_data[progress_id]["size"] = unicode(items_count) | 231 progress_data[progress_id]["size"] = str(items_count) |
232 metadata = { | 232 metadata = { |
233 "name": u"{}: {}".format(name, location), | 233 "name": "{}: {}".format(name, location), |
234 "direction": "out", | 234 "direction": "out", |
235 "type": import_handler.name.upper() + "_IMPORT", | 235 "type": import_handler.name.upper() + "_IMPORT", |
236 } | 236 } |
237 self.host.registerProgressCb( | 237 self.host.registerProgressCb( |
238 progress_id, | 238 progress_id, |
240 metadata, | 240 metadata, |
241 profile=client.profile, | 241 profile=client.profile, |
242 ) | 242 ) |
243 self.host.bridge.progressStarted(progress_id, metadata, client.profile) | 243 self.host.bridge.progressStarted(progress_id, metadata, client.profile) |
244 session = { # session data, can be used by importers | 244 session = { # session data, can be used by importers |
245 u"root_service": pubsub_service, | 245 "root_service": pubsub_service, |
246 u"root_node": pubsub_node, | 246 "root_node": pubsub_node, |
247 } | 247 } |
248 self.recursiveImport( | 248 self.recursiveImport( |
249 client, | 249 client, |
250 import_handler, | 250 import_handler, |
251 items_import_data, | 251 items_import_data, |
304 recurse_kwargs["progress_id"] = progress_id | 304 recurse_kwargs["progress_id"] = progress_id |
305 recurse_kwargs["session"] = session | 305 recurse_kwargs["session"] = session |
306 recurse_kwargs.setdefault("options", options) | 306 recurse_kwargs.setdefault("options", options) |
307 recurse_kwargs["return_data"] = return_data | 307 recurse_kwargs["return_data"] = return_data |
308 recurse_kwargs["depth"] = depth + 1 | 308 recurse_kwargs["depth"] = depth + 1 |
309 log.debug(_(u"uploading subitems")) | 309 log.debug(_("uploading subitems")) |
310 yield self.recursiveImport(**recurse_kwargs) | 310 yield self.recursiveImport(**recurse_kwargs) |
311 | 311 |
312 if depth == 0: | 312 if depth == 0: |
313 client._import[import_handler.name][progress_id]["position"] = unicode( | 313 client._import[import_handler.name][progress_id]["position"] = str( |
314 idx + 1 | 314 idx + 1 |
315 ) | 315 ) |
316 | 316 |
317 if depth == 0: | 317 if depth == 0: |
318 self.host.bridge.progressFinished(progress_id, return_data, client.profile) | 318 self.host.bridge.progressFinished(progress_id, return_data, client.profile) |
336 """ | 336 """ |
337 name = name.lower() | 337 name = name.lower() |
338 if name in import_handler.importers: | 338 if name in import_handler.importers: |
339 raise exceptions.ConflictError( | 339 raise exceptions.ConflictError( |
340 _( | 340 _( |
341 u"An {handler_name} importer with the name {name} already exist" | 341 "An {handler_name} importer with the name {name} already exist" |
342 ).format(handler_name=import_handler.name, name=name) | 342 ).format(handler_name=import_handler.name, name=name) |
343 ) | 343 ) |
344 import_handler.importers[name] = Importer(callback, short_desc, long_desc) | 344 import_handler.importers[name] = Importer(callback, short_desc, long_desc) |
345 | 345 |
346 def unregister(self, import_handler, name): | 346 def unregister(self, import_handler, name): |