Mercurial > libervia-backend

comparison src/plugins/plugin_import.py @ 2370:2c2b826b0bb3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
plugin import: node can now be specified + added a "session" dict to keep import session data: import session data are data that can be used by importer to store anything which can be useful to keep between import methods.
author Goffi <goffi@goffi.org>
date Fri, 06 Oct 2017 08:52:51 +0200
parents cdaa58e14553
children f57a8eaec8ed
comparison
equal deleted inserted replaced
2369:cdaa58e14553 2370:2c2b826b0bb3
63 log.info(_(u'initializing {name} import handler').format(name=name)) 63 log.info(_(u'initializing {name} import handler').format(name=name))
64 import_handler.name = name 64 import_handler.name = name
65 import_handler.register = partial(self.register, import_handler) 65 import_handler.register = partial(self.register, import_handler)
66 import_handler.unregister = partial(self.unregister, import_handler) 66 import_handler.unregister = partial(self.unregister, import_handler)
67 import_handler.importers = {} 67 import_handler.importers = {}
68 def _import(name, location, options, pubsub_service, profile): 68 def _import(name, location, options, pubsub_service, pubsub_node, profile):
69 return self._doImport(import_handler, name, location, options, pubsub_service, profile) 69 return self._doImport(import_handler, name, location, options, pubsub_service, pubsub_node, profile)
70 def _importList(): 70 def _importList():
71 return self.listImporters(import_handler) 71 return self.listImporters(import_handler)
72 def _importDesc(name): 72 def _importDesc(name):
73 return self.getDescription(import_handler, name) 73 return self.getDescription(import_handler, name)
74 74
75 self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}ss', out_sign='s', method=_import, async=True) 75 self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}sss', out_sign='s', method=_import, async=True)
76 self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList) 76 self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList)
77 self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc) 77 self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc)
78 78
79 def getProgress(self, import_handler, progress_id, profile): 79 def getProgress(self, import_handler, progress_id, profile):
80 client = self.host.getClient(profile) 80 client = self.host.getClient(profile)
98 handler_name = import_handler.name, 98 handler_name = import_handler.name,
99 name = name)) 99 name = name))
100 else: 100 else:
101 return importer.short_desc, importer.long_desc 101 return importer.short_desc, importer.long_desc
102 102
103 def _doImport(self, import_handler, name, location, options, pubsub_service='', profile=C.PROF_KEY_NONE): 103 def _doImport(self, import_handler, name, location, options, pubsub_service='', pubsub_node='', profile=C.PROF_KEY_NONE):
104 client = self.host.getClient(profile) 104 client = self.host.getClient(profile)
105 options = {key: unicode(value) for key, value in options.iteritems()} 105 options = {key: unicode(value) for key, value in options.iteritems()}
106 for option in import_handler.BOOL_OPTIONS: 106 for option in import_handler.BOOL_OPTIONS:
107 try: 107 try:
108 options[option] = C.bool(options[option]) 108 options[option] = C.bool(options[option])
109 except KeyError: 109 except KeyError:
110 pass 110 pass
111 return self.doImport(client, import_handler, unicode(name), unicode(location), options) 111 return self.doImport(client, import_handler, unicode(name), unicode(location), options, pubsub_service or None, pubsub_node or None)
112 112
113 @defer.inlineCallbacks 113 @defer.inlineCallbacks
114 def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None): 114 def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None, pubsub_node=None):
115 """Import data 115 """Import data
116 116
117 @param import_handler(object): instance of the import handler 117 @param import_handler(object): instance of the import handler
118 @param name(unicode): name of the importer 118 @param name(unicode): name of the importer
119 @param location(unicode): location of the data to import 119 @param location(unicode): location of the data to import
120 can be an url, a file path, or anything which make sense 120 can be an url, a file path, or anything which make sense
121 check importer description for more details 121 check importer description for more details
122 @param options(dict, None): extra options. 122 @param options(dict, None): extra options.
123 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported 123 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported
124 None to use profile's server 124 None to use profile's server
125 @param pubsub_node(unicode, None): PubSub node to use
126 None to use importer's default node
125 @return (unicode): progress id 127 @return (unicode): progress id
126 """ 128 """
127 if options is None: 129 if options is None:
128 options = {} 130 options = {}
129 else: 131 else:
155 'direction': 'out', 157 'direction': 'out',
156 'type': import_handler.name.upper() + '_IMPORT' 158 'type': import_handler.name.upper() + '_IMPORT'
157 } 159 }
158 self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile) 160 self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile)
159 self.host.bridge.progressStarted(progress_id, metadata, client.profile) 161 self.host.bridge.progressStarted(progress_id, metadata, client.profile)
160 url_redirect = {} 162 session = {} # session data, can be use by importers
161 self.recursiveImport(client, import_handler, items_import_data, progress_id, options, url_redirect) 163 self.recursiveImport(client, import_handler, items_import_data, progress_id, session, options, None, pubsub_service, pubsub_node)
162 defer.returnValue(progress_id) 164 defer.returnValue(progress_id)
163 165
164 @defer.inlineCallbacks 166 @defer.inlineCallbacks
165 def recursiveImport(self, client, import_handler, items_import_data, progress_id, options, return_data=None, service=None, node=None, depth=0): 167 def recursiveImport(self, client, import_handler, items_import_data, progress_id, session, options, return_data=None, service=None, node=None, depth=0):
166 """Do the import recursively 168 """Do the import recursively
167 169
168 @param import_handler(object): instance of the import handler 170 @param import_handler(object): instance of the import handler
169 @param items_import_data(iterable): iterable of data as specified in [register] 171 @param items_import_data(iterable): iterable of data as specified in [register]
170 @param progress_id(unicode): id of progression 172 @param progress_id(unicode): id of progression
173 @param session(dict): data for this import session
174 can be used by importer so store any useful data
171 @param options(dict): import options 175 @param options(dict): import options
172 @param return_data(dict): data to return on progressFinished 176 @param return_data(dict): data to return on progressFinished
173 @param service(jid.JID, None): PubSub service to use 177 @param service(jid.JID, None): PubSub service to use
174 @param node(unicode, None): PubSub node to use 178 @param node(unicode, None): PubSub node to use
175 @param depth(int): level of recursion 179 @param depth(int): level of recursion
176 """ 180 """
177 if return_data is None: 181 if return_data is None:
178 return_data = {} 182 return_data = {}
179 for idx, item_import_data in enumerate(items_import_data): 183 for idx, item_import_data in enumerate(items_import_data):
180 item_data = yield import_handler.importItem(client, item_import_data, options, return_data, service, node) 184 item_data = yield import_handler.importItem(client, item_import_data, session, options, return_data, service, node)
181 yield import_handler.itemFilters(client, item_data, options) 185 yield import_handler.itemFilters(client, item_data, session, options)
182 recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, options) 186 recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, session, options)
183 yield import_handler.publishItem(client, item_data, service, node) 187 yield import_handler.publishItem(client, item_data, service, node, session)
184 188
185 if recurse_kwargs is not None: 189 if recurse_kwargs is not None:
186 recurse_kwargs['client'] = client 190 recurse_kwargs['client'] = client
187 recurse_kwargs['import_handler'] = import_handler 191 recurse_kwargs['import_handler'] = import_handler
188 recurse_kwargs['progress_id'] = progress_id 192 recurse_kwargs['progress_id'] = progress_id
193 recurse_kwargs['session'] = session
189 recurse_kwargs.setdefault('options', options) 194 recurse_kwargs.setdefault('options', options)
190 recurse_kwargs['return_data'] = return_data 195 recurse_kwargs['return_data'] = return_data
191 recurse_kwargs['depth'] = depth + 1 196 recurse_kwargs['depth'] = depth + 1
192 log.debug(_(u"uploading subitems")) 197 log.debug(_(u"uploading subitems"))
193 yield self.recursiveImport(**recurse_kwargs) 198 yield self.recursiveImport(**recurse_kwargs)