Mercurial > libervia-backend

comparison sat/plugins/plugin_import.py @ 4037:524856bd7b19

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
massive refactoring to switch from camelCase to snake_case: historically, Libervia (SàT before) was using camelCase as allowed by PEP8 when using a pre-PEP8 code, to use the same coding style as in Twisted. However, snake_case is more readable and it's better to follow PEP8 best practices, so it has been decided to move on full snake_case. Because Libervia has a huge codebase, this ended with a ugly mix of camelCase and snake_case. To fix that, this patch does a big refactoring by renaming every function and method (including bridge) that are not coming from Twisted or Wokkel, to use fully snake_case. This is a massive change, and may result in some bugs.
author Goffi <goffi@goffi.org>
date Sat, 08 Apr 2023 13:54:42 +0200
parents 04283582966f
children
comparison
equal deleted inserted replaced
4036:c4464d7ae97b 4037:524856bd7b19
44 Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc")) 44 Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc"))
45 45
46 46
47 class ImportPlugin(object): 47 class ImportPlugin(object):
48 def __init__(self, host): 48 def __init__(self, host):
49 log.info(_("plugin Import initialization")) 49 log.info(_("plugin import initialization"))
50 self.host = host 50 self.host = host
51 51
52 def initialize(self, import_handler, name): 52 def initialize(self, import_handler, name):
53 """Initialize a specialized import handler 53 """Initialize a specialized import handler
54 54
55 @param import_handler(object): specialized import handler instance 55 @param import_handler(object): specialized import handler instance
56 must have the following methods: 56 must have the following methods:
57 - importItem: import a single main item (i.e. prepare data for publishing) 57 - import_item: import a single main item (i.e. prepare data for publishing)
58 - importSubitems: import sub items (i.e. items linked to main item, e.g. comments). 58 - importSubitems: import sub items (i.e. items linked to main item, e.g. comments).
59 Must return a dict with kwargs for recursiveImport if items are to be imported recursively. 59 Must return a dict with kwargs for recursive_import if items are to be imported recursively.
60 At least "items_import_data", "service" and "node" keys must be provided. 60 At least "items_import_data", "service" and "node" keys must be provided.
61 if None is returned, no recursion will be done to import subitems, but import can still be done directly by the method. 61 if None is returned, no recursion will be done to import subitems, but import can still be done directly by the method.
62 - publishItem: actualy publish an item 62 - publish_item: actualy publish an item
63 - itemFilters: modify item according to options 63 - item_filters: modify item according to options
64 @param name(unicode): import handler name 64 @param name(unicode): import handler name
65 """ 65 """
66 assert name == name.lower().strip() 66 assert name == name.lower().strip()
67 log.info(_("initializing {name} import handler").format(name=name)) 67 log.info(_("initializing {name} import handler").format(name=name))
68 import_handler.name = name 68 import_handler.name = name
69 import_handler.register = partial(self.register, import_handler) 69 import_handler.register = partial(self.register, import_handler)
70 import_handler.unregister = partial(self.unregister, import_handler) 70 import_handler.unregister = partial(self.unregister, import_handler)
71 import_handler.importers = {} 71 import_handler.importers = {}
72 72
73 def _import(name, location, options, pubsub_service, pubsub_node, profile): 73 def _import(name, location, options, pubsub_service, pubsub_node, profile):
74 return self._doImport( 74 return self._do_import(
75 import_handler, 75 import_handler,
76 name, 76 name,
77 location, 77 location,
78 options, 78 options,
79 pubsub_service, 79 pubsub_service,
80 pubsub_node, 80 pubsub_node,
81 profile, 81 profile,
82 ) 82 )
83 83
84 def _importList(): 84 def _import_list():
85 return self.listImporters(import_handler) 85 return self.list_importers(import_handler)
86 86
87 def _importDesc(name): 87 def _import_desc(name):
88 return self.getDescription(import_handler, name) 88 return self.getDescription(import_handler, name)
89 89
90 self.host.bridge.addMethod( 90 self.host.bridge.add_method(
91 name + "Import", 91 name + "import",
92 ".plugin", 92 ".plugin",
93 in_sign="ssa{ss}sss", 93 in_sign="ssa{ss}sss",
94 out_sign="s", 94 out_sign="s",
95 method=_import, 95 method=_import,
96 async_=True, 96 async_=True,
97 ) 97 )
98 self.host.bridge.addMethod( 98 self.host.bridge.add_method(
99 name + "ImportList", 99 name + "ImportList",
100 ".plugin", 100 ".plugin",
101 in_sign="", 101 in_sign="",
102 out_sign="a(ss)", 102 out_sign="a(ss)",
103 method=_importList, 103 method=_import_list,
104 ) 104 )
105 self.host.bridge.addMethod( 105 self.host.bridge.add_method(
106 name + "ImportDesc", 106 name + "ImportDesc",
107 ".plugin", 107 ".plugin",
108 in_sign="s", 108 in_sign="s",
109 out_sign="(ss)", 109 out_sign="(ss)",
110 method=_importDesc, 110 method=_import_desc,
111 ) 111 )
112 112
113 def getProgress(self, import_handler, progress_id, profile): 113 def get_progress(self, import_handler, progress_id, profile):
114 client = self.host.getClient(profile) 114 client = self.host.get_client(profile)
115 return client._import[import_handler.name][progress_id] 115 return client._import[import_handler.name][progress_id]
116 116
117 def listImporters(self, import_handler): 117 def list_importers(self, import_handler):
118 importers = list(import_handler.importers.keys()) 118 importers = list(import_handler.importers.keys())
119 importers.sort() 119 importers.sort()
120 return [ 120 return [
121 (name, import_handler.importers[name].short_desc) 121 (name, import_handler.importers[name].short_desc)
122 for name in import_handler.importers 122 for name in import_handler.importers
137 ) 137 )
138 ) 138 )
139 else: 139 else:
140 return importer.short_desc, importer.long_desc 140 return importer.short_desc, importer.long_desc
141 141
142 def _doImport(self, import_handler, name, location, options, pubsub_service="", 142 def _do_import(self, import_handler, name, location, options, pubsub_service="",
143 pubsub_node="", profile=C.PROF_KEY_NONE): 143 pubsub_node="", profile=C.PROF_KEY_NONE):
144 client = self.host.getClient(profile) 144 client = self.host.get_client(profile)
145 options = {key: str(value) for key, value in options.items()} 145 options = {key: str(value) for key, value in options.items()}
146 for option in import_handler.BOOL_OPTIONS: 146 for option in import_handler.BOOL_OPTIONS:
147 try: 147 try:
148 options[option] = C.bool(options[option]) 148 options[option] = C.bool(options[option])
149 except KeyError: 149 except KeyError:
156 except ValueError: 156 except ValueError:
157 raise exceptions.DataError( 157 raise exceptions.DataError(
158 _("invalid json option: {option}").format(option=option) 158 _("invalid json option: {option}").format(option=option)
159 ) 159 )
160 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None 160 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None
161 return self.doImport( 161 return self.do_import(
162 client, 162 client,
163 import_handler, 163 import_handler,
164 str(name), 164 str(name),
165 str(location), 165 str(location),
166 options, 166 options,
167 pubsub_service, 167 pubsub_service,
168 pubsub_node or None, 168 pubsub_node or None,
169 ) 169 )
170 170
171 @defer.inlineCallbacks 171 @defer.inlineCallbacks
172 def doImport(self, client, import_handler, name, location, options=None, 172 def do_import(self, client, import_handler, name, location, options=None,
173 pubsub_service=None, pubsub_node=None,): 173 pubsub_service=None, pubsub_node=None,):
174 """Import data 174 """import data
175 175
176 @param import_handler(object): instance of the import handler 176 @param import_handler(object): instance of the import handler
177 @param name(unicode): name of the importer 177 @param name(unicode): name of the importer
178 @param location(unicode): location of the data to import 178 @param location(unicode): location of the data to import
179 can be an url, a file path, or anything which make sense 179 can be an url, a file path, or anything which make sense
219 metadata = { 219 metadata = {
220 "name": "{}: {}".format(name, location), 220 "name": "{}: {}".format(name, location),
221 "direction": "out", 221 "direction": "out",
222 "type": import_handler.name.upper() + "_IMPORT", 222 "type": import_handler.name.upper() + "_IMPORT",
223 } 223 }
224 self.host.registerProgressCb( 224 self.host.register_progress_cb(
225 progress_id, 225 progress_id,
226 partial(self.getProgress, import_handler), 226 partial(self.get_progress, import_handler),
227 metadata, 227 metadata,
228 profile=client.profile, 228 profile=client.profile,
229 ) 229 )
230 self.host.bridge.progressStarted(progress_id, metadata, client.profile) 230 self.host.bridge.progress_started(progress_id, metadata, client.profile)
231 session = { #  session data, can be used by importers 231 session = { #  session data, can be used by importers
232 "root_service": pubsub_service, 232 "root_service": pubsub_service,
233 "root_node": pubsub_node, 233 "root_node": pubsub_node,
234 } 234 }
235 self.recursiveImport( 235 self.recursive_import(
236 client, 236 client,
237 import_handler, 237 import_handler,
238 items_import_data, 238 items_import_data,
239 progress_id, 239 progress_id,
240 session, 240 session,
244 pubsub_node, 244 pubsub_node,
245 ) 245 )
246 defer.returnValue(progress_id) 246 defer.returnValue(progress_id)
247 247
248 @defer.inlineCallbacks 248 @defer.inlineCallbacks
249 def recursiveImport( 249 def recursive_import(
250 self, 250 self,
251 client, 251 client,
252 import_handler, 252 import_handler,
253 items_import_data, 253 items_import_data,
254 progress_id, 254 progress_id,
266 @param progress_id(unicode): id of progression 266 @param progress_id(unicode): id of progression
267 @param session(dict): data for this import session 267 @param session(dict): data for this import session
268 can be used by importer so store any useful data 268 can be used by importer so store any useful data
269 "root_service" and "root_node" are set to the main pubsub service and node of the import 269 "root_service" and "root_node" are set to the main pubsub service and node of the import
270 @param options(dict): import options 270 @param options(dict): import options
271 @param return_data(dict): data to return on progressFinished 271 @param return_data(dict): data to return on progress_finished
272 @param service(jid.JID, None): PubSub service to use 272 @param service(jid.JID, None): PubSub service to use
273 @param node(unicode, None): PubSub node to use 273 @param node(unicode, None): PubSub node to use
274 @param depth(int): level of recursion 274 @param depth(int): level of recursion
275 """ 275 """
276 if return_data is None: 276 if return_data is None:
277 return_data = {} 277 return_data = {}
278 for idx, item_import_data in enumerate(items_import_data): 278 for idx, item_import_data in enumerate(items_import_data):
279 item_data = yield import_handler.importItem( 279 item_data = yield import_handler.import_item(
280 client, item_import_data, session, options, return_data, service, node 280 client, item_import_data, session, options, return_data, service, node
281 ) 281 )
282 yield import_handler.itemFilters(client, item_data, session, options) 282 yield import_handler.item_filters(client, item_data, session, options)
283 recurse_kwargs = yield import_handler.importSubItems( 283 recurse_kwargs = yield import_handler.import_sub_items(
284 client, item_import_data, item_data, session, options 284 client, item_import_data, item_data, session, options
285 ) 285 )
286 yield import_handler.publishItem(client, item_data, service, node, session) 286 yield import_handler.publish_item(client, item_data, service, node, session)
287 287
288 if recurse_kwargs is not None: 288 if recurse_kwargs is not None:
289 recurse_kwargs["client"] = client 289 recurse_kwargs["client"] = client
290 recurse_kwargs["import_handler"] = import_handler 290 recurse_kwargs["import_handler"] = import_handler
291 recurse_kwargs["progress_id"] = progress_id 291 recurse_kwargs["progress_id"] = progress_id
292 recurse_kwargs["session"] = session 292 recurse_kwargs["session"] = session
293 recurse_kwargs.setdefault("options", options) 293 recurse_kwargs.setdefault("options", options)
294 recurse_kwargs["return_data"] = return_data 294 recurse_kwargs["return_data"] = return_data
295 recurse_kwargs["depth"] = depth + 1 295 recurse_kwargs["depth"] = depth + 1
296 log.debug(_("uploading subitems")) 296 log.debug(_("uploading subitems"))
297 yield self.recursiveImport(**recurse_kwargs) 297 yield self.recursive_import(**recurse_kwargs)
298 298
299 if depth == 0: 299 if depth == 0:
300 client._import[import_handler.name][progress_id]["position"] = str( 300 client._import[import_handler.name][progress_id]["position"] = str(
301 idx + 1 301 idx + 1
302 ) 302 )
303 303
304 if depth == 0: 304 if depth == 0:
305 self.host.bridge.progressFinished(progress_id, return_data, client.profile) 305 self.host.bridge.progress_finished(progress_id, return_data, client.profile)
306 self.host.removeProgressCb(progress_id, client.profile) 306 self.host.remove_progress_cb(progress_id, client.profile)
307 del client._import[import_handler.name][progress_id] 307 del client._import[import_handler.name][progress_id]
308 308
309 def register(self, import_handler, name, callback, short_desc="", long_desc=""): 309 def register(self, import_handler, name, callback, short_desc="", long_desc=""):
310 """Register an Importer method 310 """Register an Importer method
311 311
312 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase 312 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase
313 @param callback(callable): method to call: 313 @param callback(callable): method to call:
314 the signature must be (client, location, options) (cf. [doImport]) 314 the signature must be (client, location, options) (cf. [do_import])
315 the importer must return a tuple with (items_import_data, items_count) 315 the importer must return a tuple with (items_import_data, items_count)
316 items_import_data(iterable[dict]) data specific to specialized importer 316 items_import_data(iterable[dict]) data specific to specialized importer
317 cf. importItem docstring of specialized importer for details 317 cf. import_item docstring of specialized importer for details
318 items_count (int, None) indicate the total number of items (without subitems) 318 items_count (int, None) indicate the total number of items (without subitems)
319 useful to display a progress indicator when the iterator is a generator 319 useful to display a progress indicator when the iterator is a generator
320 use None if you can't guess the total number of items 320 use None if you can't guess the total number of items
321 @param short_desc(unicode): one line description of the importer 321 @param short_desc(unicode): one line description of the importer
322 @param long_desc(unicode): long description of the importer, its options, etc. 322 @param long_desc(unicode): long description of the importer, its options, etc.