Mercurial > libervia-backend

comparison sat/plugins/plugin_import.py @ 2624:56f94936df1e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
code style reformatting using black
author Goffi <goffi@goffi.org>
date Wed, 27 Jun 2018 20:14:46 +0200
parents 26edcf3a30eb
children 003b8b4b56a7
comparison
equal deleted inserted replaced
2623:49533de4540b 2624:56f94936df1e
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. 18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 19
20 from sat.core.i18n import _ 20 from sat.core.i18n import _
21 from sat.core.constants import Const as C 21 from sat.core.constants import Const as C
22 from sat.core.log import getLogger 22 from sat.core.log import getLogger
23
23 log = getLogger(__name__) 24 log = getLogger(__name__)
24 from twisted.internet import defer 25 from twisted.internet import defer
25 from sat.core import exceptions 26 from sat.core import exceptions
26 from twisted.words.protocols.jabber import jid 27 from twisted.words.protocols.jabber import jid
27 from functools import partial 28 from functools import partial
35 C.PI_IMPORT_NAME: "IMPORT", 36 C.PI_IMPORT_NAME: "IMPORT",
36 C.PI_TYPE: C.PLUG_TYPE_IMPORT, 37 C.PI_TYPE: C.PLUG_TYPE_IMPORT,
37 C.PI_DEPENDENCIES: [], 38 C.PI_DEPENDENCIES: [],
38 C.PI_MAIN: "ImportPlugin", 39 C.PI_MAIN: "ImportPlugin",
39 C.PI_HANDLER: "no", 40 C.PI_HANDLER: "no",
40 C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers""") 41 C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers"""),
41 } 42 }
42 43
43 Importer = collections.namedtuple('Importer', ('callback', 'short_desc', 'long_desc')) 44 Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc"))
44 45
45 46
46 class ImportPlugin(object): 47 class ImportPlugin(object):
47
48 def __init__(self, host): 48 def __init__(self, host):
49 log.info(_("plugin Import initialization")) 49 log.info(_("plugin Import initialization"))
50 self.host = host 50 self.host = host
51 51
52 def initialize(self, import_handler, name): 52 def initialize(self, import_handler, name):
62 - publishItem: actualy publish an item 62 - publishItem: actualy publish an item
63 - itemFilters: modify item according to options 63 - itemFilters: modify item according to options
64 @param name(unicode): import handler name 64 @param name(unicode): import handler name
65 """ 65 """
66 assert name == name.lower().strip() 66 assert name == name.lower().strip()
67 log.info(_(u'initializing {name} import handler').format(name=name)) 67 log.info(_(u"initializing {name} import handler").format(name=name))
68 import_handler.name = name 68 import_handler.name = name
69 import_handler.register = partial(self.register, import_handler) 69 import_handler.register = partial(self.register, import_handler)
70 import_handler.unregister = partial(self.unregister, import_handler) 70 import_handler.unregister = partial(self.unregister, import_handler)
71 import_handler.importers = {} 71 import_handler.importers = {}
72
72 def _import(name, location, options, pubsub_service, pubsub_node, profile): 73 def _import(name, location, options, pubsub_service, pubsub_node, profile):
73 return self._doImport(import_handler, name, location, options, pubsub_service, pubsub_node, profile) 74 return self._doImport(
75 import_handler,
76 name,
77 location,
78 options,
79 pubsub_service,
80 pubsub_node,
81 profile,
82 )
83
74 def _importList(): 84 def _importList():
75 return self.listImporters(import_handler) 85 return self.listImporters(import_handler)
86
76 def _importDesc(name): 87 def _importDesc(name):
77 return self.getDescription(import_handler, name) 88 return self.getDescription(import_handler, name)
78 89
79 self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}sss', out_sign='s', method=_import, async=True) 90 self.host.bridge.addMethod(
80 self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList) 91 name + "Import",
81 self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc) 92 ".plugin",
93 in_sign="ssa{ss}sss",
94 out_sign="s",
95 method=_import,
96 async=True,
97 )
98 self.host.bridge.addMethod(
99 name + "ImportList",
100 ".plugin",
101 in_sign="",
102 out_sign="a(ss)",
103 method=_importList,
104 )
105 self.host.bridge.addMethod(
106 name + "ImportDesc",
107 ".plugin",
108 in_sign="s",
109 out_sign="(ss)",
110 method=_importDesc,
111 )
82 112
83 def getProgress(self, import_handler, progress_id, profile): 113 def getProgress(self, import_handler, progress_id, profile):
84 client = self.host.getClient(profile) 114 client = self.host.getClient(profile)
85 return client._import[import_handler.name][progress_id] 115 return client._import[import_handler.name][progress_id]
86 116
87 def listImporters(self, import_handler): 117 def listImporters(self, import_handler):
88 importers = import_handler.importers.keys() 118 importers = import_handler.importers.keys()
89 importers.sort() 119 importers.sort()
90 return [(name, import_handler.importers[name].short_desc) for name in import_handler.importers] 120 return [
121 (name, import_handler.importers[name].short_desc)
122 for name in import_handler.importers
123 ]
91 124
92 def getDescription(self, import_handler, name): 125 def getDescription(self, import_handler, name):
93 """Return import short and long descriptions 126 """Return import short and long descriptions
94 127
95 @param name(unicode): importer name 128 @param name(unicode): importer name
96 @return (tuple[unicode,unicode]): short and long description 129 @return (tuple[unicode,unicode]): short and long description
97 """ 130 """
98 try: 131 try:
99 importer = import_handler.importers[name] 132 importer = import_handler.importers[name]
100 except KeyError: 133 except KeyError:
101 raise exceptions.NotFound(u"{handler_name} importer not found [{name}]".format( 134 raise exceptions.NotFound(
102 handler_name = import_handler.name, 135 u"{handler_name} importer not found [{name}]".format(
103 name = name)) 136 handler_name=import_handler.name, name=name
137 )
138 )
104 else: 139 else:
105 return importer.short_desc, importer.long_desc 140 return importer.short_desc, importer.long_desc
106 141
107 def _doImport(self, import_handler, name, location, options, pubsub_service='', pubsub_node='', profile=C.PROF_KEY_NONE): 142 def _doImport(
143 self,
144 import_handler,
145 name,
146 location,
147 options,
148 pubsub_service="",
149 pubsub_node="",
150 profile=C.PROF_KEY_NONE,
151 ):
108 client = self.host.getClient(profile) 152 client = self.host.getClient(profile)
109 options = {key: unicode(value) for key, value in options.iteritems()} 153 options = {key: unicode(value) for key, value in options.iteritems()}
110 for option in import_handler.BOOL_OPTIONS: 154 for option in import_handler.BOOL_OPTIONS:
111 try: 155 try:
112 options[option] = C.bool(options[option]) 156 options[option] = C.bool(options[option])
114 pass 158 pass
115 for option in import_handler.JSON_OPTIONS: 159 for option in import_handler.JSON_OPTIONS:
116 try: 160 try:
117 options[option] = json.loads(options[option]) 161 options[option] = json.loads(options[option])
118 except ValueError: 162 except ValueError:
119 raise exceptions.DataError(_(u'invalid json option: {name}').format(name=option)) 163 raise exceptions.DataError(
164 _(u"invalid json option: {name}").format(name=option)
165 )
120 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None 166 pubsub_service = jid.JID(pubsub_service) if pubsub_service else None
121 return self.doImport(client, import_handler, unicode(name), unicode(location), options, pubsub_service, pubsub_node or None) 167 return self.doImport(
168 client,
169 import_handler,
170 unicode(name),
171 unicode(location),
172 options,
173 pubsub_service,
174 pubsub_node or None,
175 )
122 176
123 @defer.inlineCallbacks 177 @defer.inlineCallbacks
124 def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None, pubsub_node=None): 178 def doImport(
179 self,
180 client,
181 import_handler,
182 name,
183 location,
184 options=None,
185 pubsub_service=None,
186 pubsub_node=None,
187 ):
125 """Import data 188 """Import data
126 189
127 @param import_handler(object): instance of the import handler 190 @param import_handler(object): instance of the import handler
128 @param name(unicode): name of the importer 191 @param name(unicode): name of the importer
129 @param location(unicode): location of the data to import 192 @param location(unicode): location of the data to import
140 options = {} 203 options = {}
141 else: 204 else:
142 for opt_name, opt_default in import_handler.OPT_DEFAULTS.iteritems(): 205 for opt_name, opt_default in import_handler.OPT_DEFAULTS.iteritems():
143 # we want a filled options dict, with all empty or False values removed 206 # we want a filled options dict, with all empty or False values removed
144 try: 207 try:
145 value =options[opt_name] 208 value = options[opt_name]
146 except KeyError: 209 except KeyError:
147 if opt_default: 210 if opt_default:
148 options[opt_name] = opt_default 211 options[opt_name] = opt_default
149 else: 212 else:
150 if not value: 213 if not value:
152 215
153 try: 216 try:
154 importer = import_handler.importers[name] 217 importer = import_handler.importers[name]
155 except KeyError: 218 except KeyError:
156 raise exceptions.NotFound(u"Importer [{}] not found".format(name)) 219 raise exceptions.NotFound(u"Importer [{}] not found".format(name))
157 items_import_data, items_count = yield importer.callback(client, location, options) 220 items_import_data, items_count = yield importer.callback(
221 client, location, options
222 )
158 progress_id = unicode(uuid.uuid4()) 223 progress_id = unicode(uuid.uuid4())
159 try: 224 try:
160 _import = client._import 225 _import = client._import
161 except AttributeError: 226 except AttributeError:
162 _import = client._import = {} 227 _import = client._import = {}
163 progress_data = _import.setdefault(import_handler.name, {}) 228 progress_data = _import.setdefault(import_handler.name, {})
164 progress_data[progress_id] = {u'position': '0'} 229 progress_data[progress_id] = {u"position": "0"}
165 if items_count is not None: 230 if items_count is not None:
166 progress_data[progress_id]['size'] = unicode(items_count) 231 progress_data[progress_id]["size"] = unicode(items_count)
167 metadata = {'name': u'{}: {}'.format(name, location), 232 metadata = {
168 'direction': 'out', 233 "name": u"{}: {}".format(name, location),
169 'type': import_handler.name.upper() + '_IMPORT' 234 "direction": "out",
170 } 235 "type": import_handler.name.upper() + "_IMPORT",
171 self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile) 236 }
237 self.host.registerProgressCb(
238 progress_id,
239 partial(self.getProgress, import_handler),
240 metadata,
241 profile=client.profile,
242 )
172 self.host.bridge.progressStarted(progress_id, metadata, client.profile) 243 self.host.bridge.progressStarted(progress_id, metadata, client.profile)
173 session = { # session data, can be used by importers 244 session = { #  session data, can be used by importers
174 u'root_service': pubsub_service, 245 u"root_service": pubsub_service,
175 u'root_node': pubsub_node 246 u"root_node": pubsub_node,
176 } 247 }
177 self.recursiveImport(client, import_handler, items_import_data, progress_id, session, options, None, pubsub_service, pubsub_node) 248 self.recursiveImport(
249 client,
250 import_handler,
251 items_import_data,
252 progress_id,
253 session,
254 options,
255 None,
256 pubsub_service,
257 pubsub_node,
258 )
178 defer.returnValue(progress_id) 259 defer.returnValue(progress_id)
179 260
180 @defer.inlineCallbacks 261 @defer.inlineCallbacks
181 def recursiveImport(self, client, import_handler, items_import_data, progress_id, session, options, return_data=None, service=None, node=None, depth=0): 262 def recursiveImport(
263 self,
264 client,
265 import_handler,
266 items_import_data,
267 progress_id,
268 session,
269 options,
270 return_data=None,
271 service=None,
272 node=None,
273 depth=0,
274 ):
182 """Do the import recursively 275 """Do the import recursively
183 276
184 @param import_handler(object): instance of the import handler 277 @param import_handler(object): instance of the import handler
185 @param items_import_data(iterable): iterable of data as specified in [register] 278 @param items_import_data(iterable): iterable of data as specified in [register]
186 @param progress_id(unicode): id of progression 279 @param progress_id(unicode): id of progression
194 @param depth(int): level of recursion 287 @param depth(int): level of recursion
195 """ 288 """
196 if return_data is None: 289 if return_data is None:
197 return_data = {} 290 return_data = {}
198 for idx, item_import_data in enumerate(items_import_data): 291 for idx, item_import_data in enumerate(items_import_data):
199 item_data = yield import_handler.importItem(client, item_import_data, session, options, return_data, service, node) 292 item_data = yield import_handler.importItem(
293 client, item_import_data, session, options, return_data, service, node
294 )
200 yield import_handler.itemFilters(client, item_data, session, options) 295 yield import_handler.itemFilters(client, item_data, session, options)
201 recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, session, options) 296 recurse_kwargs = yield import_handler.importSubItems(
297 client, item_import_data, item_data, session, options
298 )
202 yield import_handler.publishItem(client, item_data, service, node, session) 299 yield import_handler.publishItem(client, item_data, service, node, session)
203 300
204 if recurse_kwargs is not None: 301 if recurse_kwargs is not None:
205 recurse_kwargs['client'] = client 302 recurse_kwargs["client"] = client
206 recurse_kwargs['import_handler'] = import_handler 303 recurse_kwargs["import_handler"] = import_handler
207 recurse_kwargs['progress_id'] = progress_id 304 recurse_kwargs["progress_id"] = progress_id
208 recurse_kwargs['session'] = session 305 recurse_kwargs["session"] = session
209 recurse_kwargs.setdefault('options', options) 306 recurse_kwargs.setdefault("options", options)
210 recurse_kwargs['return_data'] = return_data 307 recurse_kwargs["return_data"] = return_data
211 recurse_kwargs['depth'] = depth + 1 308 recurse_kwargs["depth"] = depth + 1
212 log.debug(_(u"uploading subitems")) 309 log.debug(_(u"uploading subitems"))
213 yield self.recursiveImport(**recurse_kwargs) 310 yield self.recursiveImport(**recurse_kwargs)
214 311
215 if depth == 0: 312 if depth == 0:
216 client._import[import_handler.name][progress_id]['position'] = unicode(idx+1) 313 client._import[import_handler.name][progress_id]["position"] = unicode(
314 idx + 1
315 )
217 316
218 if depth == 0: 317 if depth == 0:
219 self.host.bridge.progressFinished(progress_id, 318 self.host.bridge.progressFinished(progress_id, return_data, client.profile)
220 return_data,
221 client.profile)
222 self.host.removeProgressCb(progress_id, client.profile) 319 self.host.removeProgressCb(progress_id, client.profile)
223 del client._import[import_handler.name][progress_id] 320 del client._import[import_handler.name][progress_id]
224 321
225 def register(self, import_handler, name, callback, short_desc='', long_desc=''): 322 def register(self, import_handler, name, callback, short_desc="", long_desc=""):
226 """Register an Importer method 323 """Register an Importer method
227 324
228 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase 325 @param name(unicode): unique importer name, should indicate the software it can import and always lowercase
229 @param callback(callable): method to call: 326 @param callback(callable): method to call:
230 the signature must be (client, location, options) (cf. [doImport]) 327 the signature must be (client, location, options) (cf. [doImport])
237 @param short_desc(unicode): one line description of the importer 334 @param short_desc(unicode): one line description of the importer
238 @param long_desc(unicode): long description of the importer, its options, etc. 335 @param long_desc(unicode): long description of the importer, its options, etc.
239 """ 336 """
240 name = name.lower() 337 name = name.lower()
241 if name in import_handler.importers: 338 if name in import_handler.importers:
242 raise exceptions.ConflictError(_(u"An {handler_name} importer with the name {name} already exist").format( 339 raise exceptions.ConflictError(
243 handler_name = import_handler.name, 340 _(
244 name = name)) 341 u"An {handler_name} importer with the name {name} already exist"
342 ).format(handler_name=import_handler.name, name=name)
343 )
245 import_handler.importers[name] = Importer(callback, short_desc, long_desc) 344 import_handler.importers[name] = Importer(callback, short_desc, long_desc)
246 345
247 def unregister(self, import_handler, name): 346 def unregister(self, import_handler, name):
248 del import_handler.importers[name] 347 del import_handler.importers[name]