Mercurial > libervia-backend
comparison src/plugins/plugin_import.py @ 2370:2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
import session data are data that can be used by importer to store anything which can be useful to keep between import methods.
author | Goffi <goffi@goffi.org> |
---|---|
date | Fri, 06 Oct 2017 08:52:51 +0200 |
parents | cdaa58e14553 |
children | f57a8eaec8ed |
comparison
equal
deleted
inserted
replaced
2369:cdaa58e14553 | 2370:2c2b826b0bb3 |
---|---|
63 log.info(_(u'initializing {name} import handler').format(name=name)) | 63 log.info(_(u'initializing {name} import handler').format(name=name)) |
64 import_handler.name = name | 64 import_handler.name = name |
65 import_handler.register = partial(self.register, import_handler) | 65 import_handler.register = partial(self.register, import_handler) |
66 import_handler.unregister = partial(self.unregister, import_handler) | 66 import_handler.unregister = partial(self.unregister, import_handler) |
67 import_handler.importers = {} | 67 import_handler.importers = {} |
68 def _import(name, location, options, pubsub_service, profile): | 68 def _import(name, location, options, pubsub_service, pubsub_node, profile): |
69 return self._doImport(import_handler, name, location, options, pubsub_service, profile) | 69 return self._doImport(import_handler, name, location, options, pubsub_service, pubsub_node, profile) |
70 def _importList(): | 70 def _importList(): |
71 return self.listImporters(import_handler) | 71 return self.listImporters(import_handler) |
72 def _importDesc(name): | 72 def _importDesc(name): |
73 return self.getDescription(import_handler, name) | 73 return self.getDescription(import_handler, name) |
74 | 74 |
75 self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}ss', out_sign='s', method=_import, async=True) | 75 self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}sss', out_sign='s', method=_import, async=True) |
76 self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList) | 76 self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList) |
77 self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc) | 77 self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc) |
78 | 78 |
79 def getProgress(self, import_handler, progress_id, profile): | 79 def getProgress(self, import_handler, progress_id, profile): |
80 client = self.host.getClient(profile) | 80 client = self.host.getClient(profile) |
98 handler_name = import_handler.name, | 98 handler_name = import_handler.name, |
99 name = name)) | 99 name = name)) |
100 else: | 100 else: |
101 return importer.short_desc, importer.long_desc | 101 return importer.short_desc, importer.long_desc |
102 | 102 |
103 def _doImport(self, import_handler, name, location, options, pubsub_service='', profile=C.PROF_KEY_NONE): | 103 def _doImport(self, import_handler, name, location, options, pubsub_service='', pubsub_node='', profile=C.PROF_KEY_NONE): |
104 client = self.host.getClient(profile) | 104 client = self.host.getClient(profile) |
105 options = {key: unicode(value) for key, value in options.iteritems()} | 105 options = {key: unicode(value) for key, value in options.iteritems()} |
106 for option in import_handler.BOOL_OPTIONS: | 106 for option in import_handler.BOOL_OPTIONS: |
107 try: | 107 try: |
108 options[option] = C.bool(options[option]) | 108 options[option] = C.bool(options[option]) |
109 except KeyError: | 109 except KeyError: |
110 pass | 110 pass |
111 return self.doImport(client, import_handler, unicode(name), unicode(location), options) | 111 return self.doImport(client, import_handler, unicode(name), unicode(location), options, pubsub_service or None, pubsub_node or None) |
112 | 112 |
113 @defer.inlineCallbacks | 113 @defer.inlineCallbacks |
114 def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None): | 114 def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None, pubsub_node=None): |
115 """Import data | 115 """Import data |
116 | 116 |
117 @param import_handler(object): instance of the import handler | 117 @param import_handler(object): instance of the import handler |
118 @param name(unicode): name of the importer | 118 @param name(unicode): name of the importer |
119 @param location(unicode): location of the data to import | 119 @param location(unicode): location of the data to import |
120 can be an url, a file path, or anything which make sense | 120 can be an url, a file path, or anything which make sense |
121 check importer description for more details | 121 check importer description for more details |
122 @param options(dict, None): extra options. | 122 @param options(dict, None): extra options. |
123 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported | 123 @param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported |
124 None to use profile's server | 124 None to use profile's server |
125 @param pubsub_node(unicode, None): PubSub node to use | |
126 None to use importer's default node | |
125 @return (unicode): progress id | 127 @return (unicode): progress id |
126 """ | 128 """ |
127 if options is None: | 129 if options is None: |
128 options = {} | 130 options = {} |
129 else: | 131 else: |
155 'direction': 'out', | 157 'direction': 'out', |
156 'type': import_handler.name.upper() + '_IMPORT' | 158 'type': import_handler.name.upper() + '_IMPORT' |
157 } | 159 } |
158 self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile) | 160 self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile) |
159 self.host.bridge.progressStarted(progress_id, metadata, client.profile) | 161 self.host.bridge.progressStarted(progress_id, metadata, client.profile) |
160 url_redirect = {} | 162 session = {} # session data, can be use by importers |
161 self.recursiveImport(client, import_handler, items_import_data, progress_id, options, url_redirect) | 163 self.recursiveImport(client, import_handler, items_import_data, progress_id, session, options, None, pubsub_service, pubsub_node) |
162 defer.returnValue(progress_id) | 164 defer.returnValue(progress_id) |
163 | 165 |
164 @defer.inlineCallbacks | 166 @defer.inlineCallbacks |
165 def recursiveImport(self, client, import_handler, items_import_data, progress_id, options, return_data=None, service=None, node=None, depth=0): | 167 def recursiveImport(self, client, import_handler, items_import_data, progress_id, session, options, return_data=None, service=None, node=None, depth=0): |
166 """Do the import recursively | 168 """Do the import recursively |
167 | 169 |
168 @param import_handler(object): instance of the import handler | 170 @param import_handler(object): instance of the import handler |
169 @param items_import_data(iterable): iterable of data as specified in [register] | 171 @param items_import_data(iterable): iterable of data as specified in [register] |
170 @param progress_id(unicode): id of progression | 172 @param progress_id(unicode): id of progression |
173 @param session(dict): data for this import session | |
174 can be used by importer so store any useful data | |
171 @param options(dict): import options | 175 @param options(dict): import options |
172 @param return_data(dict): data to return on progressFinished | 176 @param return_data(dict): data to return on progressFinished |
173 @param service(jid.JID, None): PubSub service to use | 177 @param service(jid.JID, None): PubSub service to use |
174 @param node(unicode, None): PubSub node to use | 178 @param node(unicode, None): PubSub node to use |
175 @param depth(int): level of recursion | 179 @param depth(int): level of recursion |
176 """ | 180 """ |
177 if return_data is None: | 181 if return_data is None: |
178 return_data = {} | 182 return_data = {} |
179 for idx, item_import_data in enumerate(items_import_data): | 183 for idx, item_import_data in enumerate(items_import_data): |
180 item_data = yield import_handler.importItem(client, item_import_data, options, return_data, service, node) | 184 item_data = yield import_handler.importItem(client, item_import_data, session, options, return_data, service, node) |
181 yield import_handler.itemFilters(client, item_data, options) | 185 yield import_handler.itemFilters(client, item_data, session, options) |
182 recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, options) | 186 recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, session, options) |
183 yield import_handler.publishItem(client, item_data, service, node) | 187 yield import_handler.publishItem(client, item_data, service, node, session) |
184 | 188 |
185 if recurse_kwargs is not None: | 189 if recurse_kwargs is not None: |
186 recurse_kwargs['client'] = client | 190 recurse_kwargs['client'] = client |
187 recurse_kwargs['import_handler'] = import_handler | 191 recurse_kwargs['import_handler'] = import_handler |
188 recurse_kwargs['progress_id'] = progress_id | 192 recurse_kwargs['progress_id'] = progress_id |
193 recurse_kwargs['session'] = session | |
189 recurse_kwargs.setdefault('options', options) | 194 recurse_kwargs.setdefault('options', options) |
190 recurse_kwargs['return_data'] = return_data | 195 recurse_kwargs['return_data'] = return_data |
191 recurse_kwargs['depth'] = depth + 1 | 196 recurse_kwargs['depth'] = depth + 1 |
192 log.debug(_(u"uploading subitems")) | 197 log.debug(_(u"uploading subitems")) |
193 yield self.recursiveImport(**recurse_kwargs) | 198 yield self.recursiveImport(**recurse_kwargs) |