Mercurial > libervia-backend
comparison sat/plugins/plugin_misc_app_manager.py @ 3998:402d31527af4
plugin app manager: `start` doesn't wait anymore for actual app start:
Application may be long to start (e.g. a Docker app may have to download images first,
and even without the downloading, the starting could be long), which may lead to UI
blocking or bridge time out.
To prevent that, `start` is now returning immediately, and 2 new signals are used to
indicate when the application is started, of if something wrong happened.
`start` now returns initial app data, including exposed data without the computed exposed
data. The computed data must be retrieved after the app has been started.
author | Goffi <goffi@goffi.org> |
---|---|
date | Sat, 04 Mar 2023 18:30:47 +0100 |
parents | d66a8453b02b |
children | 524856bd7b19 |
comparison
equal
deleted
inserted
replaced
3997:1b7c6ee080b9 | 3998:402d31527af4 |
---|---|
1 #!/usr/bin/env python3 | 1 #!/usr/bin/env python3 |
2 | 2 |
3 # SàT plugin to manage external applications | 3 # Libervia plugin to manage external applications |
4 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) | 4 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) |
5 | 5 |
6 # This program is free software: you can redistribute it and/or modify | 6 # This program is free software: you can redistribute it and/or modify |
7 # it under the terms of the GNU Affero General Public License as published by | 7 # it under the terms of the GNU Affero General Public License as published by |
8 # the Free Software Foundation, either version 3 of the License, or | 8 # the Free Software Foundation, either version 3 of the License, or |
15 | 15 |
16 # You should have received a copy of the GNU Affero General Public License | 16 # You should have received a copy of the GNU Affero General Public License |
17 # along with this program. If not, see <http://www.gnu.org/licenses/>. | 17 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
18 | 18 |
19 from pathlib import Path | 19 from pathlib import Path |
20 from typing import Optional, List | 20 from typing import Optional, List, Callable |
21 from functools import partial, reduce | 21 from functools import partial, reduce |
22 import tempfile | 22 import tempfile |
23 import secrets | 23 import secrets |
24 import string | 24 import string |
25 import shortuuid | 25 import shortuuid |
93 ) | 93 ) |
94 host.bridge.addMethod( | 94 host.bridge.addMethod( |
95 "applicationStart", | 95 "applicationStart", |
96 ".plugin", | 96 ".plugin", |
97 in_sign="ss", | 97 in_sign="ss", |
98 out_sign="", | 98 out_sign="s", |
99 method=self._start, | 99 method=self._start, |
100 async_=True, | 100 async_=True, |
101 ) | 101 ) |
102 host.bridge.addMethod( | 102 host.bridge.addMethod( |
103 "applicationStop", | 103 "applicationStop", |
110 host.bridge.addMethod( | 110 host.bridge.addMethod( |
111 "applicationExposedGet", | 111 "applicationExposedGet", |
112 ".plugin", | 112 ".plugin", |
113 in_sign="sss", | 113 in_sign="sss", |
114 out_sign="s", | 114 out_sign="s", |
115 method=self._getExposed, | 115 method=self._get_exposed, |
116 async_=True, | 116 async_=True, |
117 ) | |
118 # application has been started succeesfully, | |
119 # args: name, instance_id, extra | |
120 host.bridge.addSignal( | |
121 "application_started", ".plugin", signature="sss" | |
122 ) | |
123 # application went wrong with the application | |
124 # args: name, instance_id, extra | |
125 host.bridge.addSignal( | |
126 "application_error", ".plugin", signature="sss" | |
117 ) | 127 ) |
118 yaml.add_constructor( | 128 yaml.add_constructor( |
119 "!sat_conf", self._sat_conf_constr, Loader=Loader) | 129 "!sat_conf", self._sat_conf_constr, Loader=Loader) |
120 yaml.add_constructor( | 130 yaml.add_constructor( |
121 "!sat_generate_pwd", self._sat_generate_pwd_constr, Loader=Loader) | 131 "!sat_generate_pwd", self._sat_generate_pwd_constr, Loader=Loader) |
131 log.debug( | 141 log.debug( |
132 f"cleaning temporary directory at {data['_instance_dir_path']}") | 142 f"cleaning temporary directory at {data['_instance_dir_path']}") |
133 data['_instance_dir_obj'].cleanup() | 143 data['_instance_dir_obj'].cleanup() |
134 | 144 |
135 def _sat_conf_constr(self, loader, node): | 145 def _sat_conf_constr(self, loader, node): |
136 """Get a value from SàT configuration | 146 """Get a value from Libervia configuration |
137 | 147 |
138 A list is expected with either "name" of a config parameter, a one or more of | 148 A list is expected with either "name" of a config parameter, a one or more of |
139 those parameters: | 149 those parameters: |
140 - section | 150 - section |
141 - name | 151 - name |
206 f"There is already a manager with the name {name}") | 216 f"There is already a manager with the name {name}") |
207 self._managers[manager.name] = manager | 217 self._managers[manager.name] = manager |
208 if hasattr(manager, "discover_path"): | 218 if hasattr(manager, "discover_path"): |
209 self.discover(manager.discover_path, manager) | 219 self.discover(manager.discover_path, manager) |
210 | 220 |
211 def getManager(self, app_data: dict) -> object: | 221 def get_manager(self, app_data: dict) -> object: |
212 """Get manager instance needed for this app | 222 """Get manager instance needed for this app |
213 | 223 |
214 @raise exceptions.DataError: something is wrong with the type | 224 @raise exceptions.DataError: something is wrong with the type |
215 @raise exceptions.NotFound: manager is not registered | 225 @raise exceptions.NotFound: manager is not registered |
216 """ | 226 """ |
229 return self._managers[app_type] | 239 return self._managers[app_type] |
230 except KeyError: | 240 except KeyError: |
231 raise exceptions.NotFound( | 241 raise exceptions.NotFound( |
232 f"No manager found to manage app of type {app_type!r}") | 242 f"No manager found to manage app of type {app_type!r}") |
233 | 243 |
234 def getAppData( | 244 def get_app_data( |
235 self, | 245 self, |
236 id_type: Optional[str], | 246 id_type: Optional[str], |
237 identifier: str | 247 identifier: str |
238 ) -> dict: | 248 ) -> dict: |
239 """Retrieve instance's app_data from identifier | 249 """Retrieve instance's app_data from identifier |
275 ) -> None: | 285 ) -> None: |
276 for file_path in dir_path.glob(f"{APP_FILE_PREFIX}*.yaml"): | 286 for file_path in dir_path.glob(f"{APP_FILE_PREFIX}*.yaml"): |
277 if manager is None: | 287 if manager is None: |
278 try: | 288 try: |
279 app_data = self.parse(file_path) | 289 app_data = self.parse(file_path) |
280 manager = self.getManager(app_data) | 290 manager = self.get_manager(app_data) |
281 except (exceptions.DataError, exceptions.NotFound) as e: | 291 except (exceptions.DataError, exceptions.NotFound) as e: |
282 log.warning( | 292 log.warning( |
283 f"Can't parse {file_path}, skipping: {e}") | 293 f"Can't parse {file_path}, skipping: {e}") |
284 app_name = file_path.stem[len(APP_FILE_PREFIX):].strip().lower() | 294 app_name = file_path.stem[len(APP_FILE_PREFIX):].strip().lower() |
285 if not app_name: | 295 if not app_name: |
292 log.debug( | 302 log.debug( |
293 f"{app_name!r} {manager.name} application found" | 303 f"{app_name!r} {manager.name} application found" |
294 ) | 304 ) |
295 | 305 |
296 def parse(self, file_path: Path, params: Optional[dict] = None) -> dict: | 306 def parse(self, file_path: Path, params: Optional[dict] = None) -> dict: |
297 """Parse SàT application file | 307 """Parse Libervia application file |
298 | 308 |
299 @param params: parameters for running this instance | 309 @param params: parameters for running this instance |
300 @raise exceptions.DataError: something is wrong in the file | 310 @raise exceptions.DataError: something is wrong in the file |
301 """ | 311 """ |
302 if params is None: | 312 if params is None: |
339 raise ValueError(f"Unknown filter: {filter_}") | 349 raise ValueError(f"Unknown filter: {filter_}") |
340 return list(found) | 350 return list(found) |
341 | 351 |
342 def _start(self, app_name, extra): | 352 def _start(self, app_name, extra): |
343 extra = data_format.deserialise(extra) | 353 extra = data_format.deserialise(extra) |
344 return defer.ensureDeferred(self.start(str(app_name), extra)) | 354 d = defer.ensureDeferred(self.start(str(app_name), extra)) |
355 d.addCallback(data_format.serialise) | |
356 return d | |
345 | 357 |
346 async def start( | 358 async def start( |
347 self, | 359 self, |
348 app_name: str, | 360 app_name: str, |
349 extra: Optional[dict] = None, | 361 extra: Optional[dict] = None, |
350 ) -> None: | 362 ) -> dict: |
363 """Start an application | |
364 | |
365 @param app_name: name of the application to start | |
366 @param extra: extra parameters | |
367 @return: data with following keys: | |
368 - name (str): canonical application name | |
369 - instance (str): instance ID | |
370 - started (bool): True if the application is already started | |
371 if False, the "application_started" signal should be used to get notificed | |
372 when the application is actually started | |
373 - expose (dict): exposed data as given by [self.get_exposed] | |
374 exposed data which need to be computed are NOT returned, they will | |
375 available when the app will be fully started, throught the | |
376 [self.get_exposed] method. | |
377 """ | |
351 # FIXME: for now we use the first app manager available for the requested app_name | 378 # FIXME: for now we use the first app manager available for the requested app_name |
352 # TODO: implement running multiple instance of the same app if some metadata | 379 # TODO: implement running multiple instance of the same app if some metadata |
353 # to be defined in app_data allows explicitly it. | 380 # to be defined in app_data allows explicitly it. |
354 app_name = app_name.lower().strip() | 381 app_name = app_name.lower().strip() |
355 try: | 382 try: |
356 app_file_path = next(iter(next(iter(self._apps[app_name].values())))) | 383 app_file_path = next(iter(next(iter(self._apps[app_name].values())))) |
357 except KeyError: | 384 except KeyError: |
358 raise exceptions.NotFound( | 385 raise exceptions.NotFound( |
359 f"No application found with the name {app_name!r}" | 386 f"No application found with the name {app_name!r}" |
360 ) | 387 ) |
388 log.info(f"starting {app_name!r}") | |
361 started_data = self._started.setdefault(app_name, []) | 389 started_data = self._started.setdefault(app_name, []) |
362 app_data = self.parse(app_file_path, extra) | 390 app_data = self.parse(app_file_path, extra) |
391 app_data["_started"] = False | |
363 app_data['_file_path'] = app_file_path | 392 app_data['_file_path'] = app_file_path |
364 app_data['_name_canonical'] = app_name | 393 app_data['_name_canonical'] = app_name |
365 single_instance = app_data['single_instance'] | 394 single_instance = app_data['single_instance'] |
395 ret_data = { | |
396 "name": app_name, | |
397 "started": False | |
398 } | |
366 if single_instance: | 399 if single_instance: |
367 if started_data: | 400 if started_data: |
368 log.info(f"{app_name!r} is already started") | 401 instance_data = started_data[0] |
369 return | 402 instance_id = instance_data["_instance_id"] |
403 ret_data["instance"] = instance_id | |
404 ret_data["started"] = instance_data["_started"] | |
405 ret_data["expose"] = await self.get_exposed( | |
406 instance_id, "instance", {"skip_compute": True} | |
407 ) | |
408 log.info(f"{app_name!r} is already started or being started") | |
409 return ret_data | |
370 else: | 410 else: |
371 cache_path = self.host.memory.getCachePath( | 411 cache_path = self.host.memory.getCachePath( |
372 PLUGIN_INFO[C.PI_IMPORT_NAME], app_name | 412 PLUGIN_INFO[C.PI_IMPORT_NAME], app_name |
373 ) | 413 ) |
374 cache_path.mkdir(0o700, parents=True, exist_ok=True) | 414 cache_path.mkdir(0o700, parents=True, exist_ok=True) |
375 app_data['_instance_dir_path'] = cache_path | 415 app_data['_instance_dir_path'] = cache_path |
376 else: | 416 else: |
377 dest_dir_obj = tempfile.TemporaryDirectory(prefix="sat_app_") | 417 dest_dir_obj = tempfile.TemporaryDirectory(prefix="sat_app_") |
378 app_data['_instance_dir_obj'] = dest_dir_obj | 418 app_data['_instance_dir_obj'] = dest_dir_obj |
379 app_data['_instance_dir_path'] = Path(dest_dir_obj.name) | 419 app_data['_instance_dir_path'] = Path(dest_dir_obj.name) |
380 instance_id = app_data['_instance_id'] = shortuuid.uuid() | 420 instance_id = ret_data["instance"] = app_data['_instance_id'] = shortuuid.uuid() |
381 manager = self.getManager(app_data) | 421 manager = self.get_manager(app_data) |
382 app_data['_manager'] = manager | 422 app_data['_manager'] = manager |
383 started_data.append(app_data) | 423 started_data.append(app_data) |
384 self._instances[instance_id] = app_data | 424 self._instances[instance_id] = app_data |
425 # we retrieve exposed data such as url_prefix which can be useful computed exposed | |
426 # data must wait for the app to be started, so we skip them for now | |
427 ret_data["expose"] = await self.get_exposed( | |
428 instance_id, "instance", {"skip_compute": True} | |
429 ) | |
385 | 430 |
386 try: | 431 try: |
387 start = manager.start | 432 start = manager.start |
388 except AttributeError: | 433 except AttributeError: |
389 raise exceptions.InternalError( | 434 raise exceptions.InternalError( |
390 f"{manager.name} doesn't have the mandatory \"start\" method" | 435 f"{manager.name} doesn't have the mandatory \"start\" method" |
391 ) | 436 ) |
392 else: | 437 else: |
393 await start(app_data) | 438 defer.ensureDeferred(self.start_app(start, app_data)) |
394 log.info(f"{app_name!r} started") | 439 return ret_data |
440 | |
441 async def start_app(self, start_cb: Callable, app_data: dict) -> None: | |
442 app_name = app_data["_name_canonical"] | |
443 instance_id = app_data["_instance_id"] | |
444 try: | |
445 await start_cb(app_data) | |
446 except Exception as e: | |
447 log.exception(f"Can't start libervia app {app_name!r}") | |
448 self.host.bridge.application_error( | |
449 app_name, | |
450 instance_id, | |
451 data_format.serialise( | |
452 { | |
453 "class": str(type(e)), | |
454 "msg": str(e) | |
455 } | |
456 )) | |
457 else: | |
458 app_data["_started"] = True | |
459 self.host.bridge.application_started(app_name, instance_id, "") | |
460 log.info(f"{app_name!r} started") | |
395 | 461 |
396 def _stop(self, identifier, id_type, extra): | 462 def _stop(self, identifier, id_type, extra): |
397 extra = data_format.deserialise(extra) | 463 extra = data_format.deserialise(extra) |
398 return defer.ensureDeferred( | 464 return defer.ensureDeferred( |
399 self.stop(str(identifier), str(id_type) or None, extra)) | 465 self.stop(str(identifier), str(id_type) or None, extra)) |
405 extra: Optional[dict] = None, | 471 extra: Optional[dict] = None, |
406 ) -> None: | 472 ) -> None: |
407 if extra is None: | 473 if extra is None: |
408 extra = {} | 474 extra = {} |
409 | 475 |
410 app_data = self.getAppData(id_type, identifier) | 476 app_data = self.get_app_data(id_type, identifier) |
411 | 477 |
412 log.info(f"stopping {app_data['name']!r}") | 478 log.info(f"stopping {app_data['name']!r}") |
413 | 479 |
414 app_name = app_data['_name_canonical'] | 480 app_name = app_data['_name_canonical'] |
415 instance_id = app_data['_instance_id'] | 481 instance_id = app_data['_instance_id'] |
445 f"{instance_id!r}" | 511 f"{instance_id!r}" |
446 ) | 512 ) |
447 | 513 |
448 log.info(f"{app_name!r} stopped") | 514 log.info(f"{app_name!r} stopped") |
449 | 515 |
450 def _getExposed(self, identifier, id_type, extra): | 516 def _get_exposed(self, identifier, id_type, extra): |
451 extra = data_format.deserialise(extra) | 517 extra = data_format.deserialise(extra) |
452 d = defer.ensureDeferred(self.getExposed(identifier, id_type, extra)) | 518 d = defer.ensureDeferred(self.get_exposed(identifier, id_type, extra)) |
453 d.addCallback(lambda d: data_format.serialise(d)) | 519 d.addCallback(lambda d: data_format.serialise(d)) |
454 return d | 520 return d |
455 | 521 |
456 def getValueFromPath(self, app_data: dict, path: List[str]) -> any: | 522 async def get_exposed( |
457 """Retrieve a value set in the data from it path | |
458 | |
459 @param path: list of key to use in app_data to retrieve the value | |
460 @return: found value | |
461 @raise NotFound: the value can't be found | |
462 """ | |
463 | |
464 async def getExposed( | |
465 self, | 523 self, |
466 identifier: str, | 524 identifier: str, |
467 id_type: Optional[str] = None, | 525 id_type: Optional[str] = None, |
468 extra: Optional[dict] = None, | 526 extra: Optional[dict] = None, |
469 ) -> dict: | 527 ) -> dict: |
470 """Get data exposed by the application | 528 """Get data exposed by the application |
471 | 529 |
472 The manager's "computeExpose" method will be called if it exists. It can be used | 530 The manager's "compute_expose" method will be called if it exists. It can be used |
473 to handle manager specific conventions. | 531 to handle manager specific conventions. |
474 """ | 532 """ |
475 app_data = self.getAppData(id_type, identifier) | 533 app_data = self.get_app_data(id_type, identifier) |
476 if app_data.get('_exposed_computed', False): | 534 if app_data.get('_exposed_computed', False): |
477 return app_data['expose'] | 535 return app_data['expose'] |
478 if extra is None: | 536 if extra is None: |
479 extra = {} | 537 extra = {} |
480 expose = app_data.setdefault("expose", {}) | 538 expose = app_data.setdefault("expose", {}) |
498 except Exception as e: | 556 except Exception as e: |
499 log.warning( | 557 log.warning( |
500 f"Can't retrieve exposed value for url_prefix: {e}") | 558 f"Can't retrieve exposed value for url_prefix: {e}") |
501 del expose["url_prefix"] | 559 del expose["url_prefix"] |
502 | 560 |
503 try: | 561 if extra.get("skip_compute", False): |
504 computeExpose = app_data['_manager'].computeExpose | 562 return expose |
563 | |
564 try: | |
565 compute_expose = app_data['_manager'].compute_expose | |
505 except AttributeError: | 566 except AttributeError: |
506 pass | 567 pass |
507 else: | 568 else: |
508 await computeExpose(app_data) | 569 await compute_expose(app_data) |
509 | 570 |
510 app_data['_exposed_computed'] = True | 571 app_data['_exposed_computed'] = True |
511 return expose | 572 return expose |
512 | 573 |
513 async def _doPrepare( | 574 async def _do_prepare( |
514 self, | 575 self, |
515 app_data: dict, | 576 app_data: dict, |
516 ) -> None: | 577 ) -> None: |
517 name = app_data['name'] | 578 name = app_data['name'] |
518 dest_path = app_data['_instance_dir_path'] | 579 dest_path = app_data['_instance_dir_path'] |
546 del prepare[action] | 607 del prepare[action] |
547 | 608 |
548 if prepare: | 609 if prepare: |
549 raise exceptions.InternalError('"prepare" should be empty') | 610 raise exceptions.InternalError('"prepare" should be empty') |
550 | 611 |
551 async def _doCreateFiles( | 612 async def _do_create_files( |
552 self, | 613 self, |
553 app_data: dict, | 614 app_data: dict, |
554 ) -> None: | 615 ) -> None: |
555 dest_path = app_data['_instance_dir_path'] | 616 dest_path = app_data['_instance_dir_path'] |
556 files = app_data.get('files') | 617 files = app_data.get('files') |
564 log.info(f"{path} already exists, skipping") | 625 log.info(f"{path} already exists, skipping") |
565 with path.open("w") as f: | 626 with path.open("w") as f: |
566 f.write(data.get("content", "")) | 627 f.write(data.get("content", "")) |
567 log.debug(f"{path} created") | 628 log.debug(f"{path} created") |
568 | 629 |
569 async def startCommon(self, app_data: dict) -> None: | 630 async def start_common(self, app_data: dict) -> None: |
570 """Method running common action when starting a manager | 631 """Method running common action when starting a manager |
571 | 632 |
572 It should be called by managers in "start" method. | 633 It should be called by managers in "start" method. |
573 """ | 634 """ |
574 log.info(f"starting {app_data['name']!r}") | 635 await self._do_prepare(app_data) |
575 await self._doPrepare(app_data) | 636 await self._do_create_files(app_data) |
576 await self._doCreateFiles(app_data) |