Mercurial > libervia-backend
annotate sat/plugins/plugin_misc_app_manager.py @ 3551:0ce37cf3fbf4
memory(sqlite): fix encoding when loading pickled data
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Wed, 09 Jun 2021 10:01:56 +0200 |
| parents | be6d91572633 |
| children | 2c9e95796371 |
| rev | line source |
|---|---|
| 3372 | 1 #!/usr/bin/env python3 |
| 2 | |
| 3 # SàT plugin to manage external applications | |
| 3479 | 4 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) |
| 3372 | 5 |
| 6 # This program is free software: you can redistribute it and/or modify | |
| 7 # it under the terms of the GNU Affero General Public License as published by | |
| 8 # the Free Software Foundation, either version 3 of the License, or | |
| 9 # (at your option) any later version. | |
| 10 | |
| 11 # This program is distributed in the hope that it will be useful, | |
| 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| 14 # GNU Affero General Public License for more details. | |
| 15 | |
| 16 # You should have received a copy of the GNU Affero General Public License | |
| 17 # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| 18 | |
| 19 from pathlib import Path | |
| 20 from typing import Optional, List | |
| 21 from functools import partial, reduce | |
| 22 import tempfile | |
| 23 import secrets | |
| 24 import string | |
| 25 import shortuuid | |
| 26 from twisted.internet import defer | |
| 27 from twisted.python.procutils import which | |
| 28 from sat.core.i18n import _ | |
| 29 from sat.core import exceptions | |
| 30 from sat.core.constants import Const as C | |
| 31 from sat.core.log import getLogger | |
| 32 from sat.tools.common import data_format | |
| 33 from sat.tools.common import async_process | |
| 34 | |
| 35 log = getLogger(__name__) | |
| 36 | |
| 37 try: | |
| 38 import yaml | |
| 39 except ImportError: | |
| 40 raise exceptions.MissingModule( | |
| 41 'Missing module PyYAML, please download/install it. You can use ' | |
| 42 '"pip install pyyaml"' | |
| 43 ) | |
| 44 | |
| 45 try: | |
| 46 from yaml import CLoader as Loader, CDumper as Dumper | |
| 47 except ImportError: | |
| 48 log.warning( | |
| 49 "Can't use LibYAML binding (is libyaml installed?), pure Python version will be " | |
| 50 "used, but it is slower" | |
| 51 ) | |
| 52 from yaml import Loader, Dumper | |
| 53 | |
| 54 from yaml.constructor import ConstructorError | |
| 55 | |
| 56 | |
| 57 PLUGIN_INFO = { | |
| 58 C.PI_NAME: "Applications Manager", | |
| 59 C.PI_IMPORT_NAME: "APP_MANAGER", | |
| 60 C.PI_TYPE: C.PLUG_TYPE_MISC, | |
| 61 C.PI_MODES: C.PLUG_MODE_BOTH, | |
| 62 C.PI_MAIN: "AppManager", | |
| 63 C.PI_HANDLER: "no", | |
| 64 C.PI_DESCRIPTION: _( | |
| 65 """Applications Manager | |
| 66 | |
| 67 Manage external applications using packagers, OS virtualization/containers or other | |
| 68 software management tools. | |
| 69 """), | |
| 70 } | |
| 71 | |
| 72 APP_FILE_PREFIX = "sat_app_" | |
| 73 | |
| 74 | |
| 75 class AppManager: | |
| 76 load = partial(yaml.load, Loader=Loader) | |
| 77 dump = partial(yaml.dump, Dumper=Dumper) | |
| 78 | |
| 79 def __init__(self, host): | |
| 80 log.info(_("plugin Applications Manager initialization")) | |
| 81 self.host = host | |
| 82 self._managers = {} | |
| 83 self._apps = {} | |
| 84 self._started = {} | |
| 85 # instance id to app data map | |
| 86 self._instances = {} | |
| 87 host.bridge.addMethod( | |
| 88 "applicationsList", | |
| 89 ".plugin", | |
| 90 in_sign="as", | |
| 91 out_sign="as", | |
| 92 method=self.list_applications, | |
| 93 ) | |
| 94 host.bridge.addMethod( | |
| 95 "applicationStart", | |
| 96 ".plugin", | |
| 97 in_sign="ss", | |
| 98 out_sign="", | |
| 99 method=self._start, | |
| 100 async_=True, | |
| 101 ) | |
| 102 host.bridge.addMethod( | |
| 103 "applicationStop", | |
| 104 ".plugin", | |
| 105 in_sign="sss", | |
| 106 out_sign="", | |
| 107 method=self._stop, | |
| 108 async_=True, | |
| 109 ) | |
| 110 host.bridge.addMethod( | |
| 111 "applicationExposedGet", | |
| 112 ".plugin", | |
| 113 in_sign="sss", | |
| 114 out_sign="s", | |
| 115 method=self._getExposed, | |
| 116 async_=True, | |
| 117 ) | |
| 118 yaml.add_constructor( | |
| 119 "!sat_conf", self._sat_conf_constr, Loader=Loader) | |
| 120 yaml.add_constructor( | |
| 121 "!sat_generate_pwd", self._sat_generate_pwd_constr, Loader=Loader) | |
| 122 yaml.add_constructor( | |
| 123 "!sat_param", self._sat_param_constr, Loader=Loader) | |
| 124 | |
| 125 def unload(self): | |
| 126 log.debug("unloading applications manager") | |
| 127 for instances in self._started.values(): | |
| 128 for instance in instances: | |
| 129 data = instance['data'] | |
| 130 if not data['single_instance']: | |
| 131 log.debug( | |
| 132 f"cleaning temporary directory at {data['_instance_dir_path']}") | |
| 133 data['_instance_dir_obj'].cleanup() | |
| 134 | |
| 135 def _sat_conf_constr(self, loader, node): | |
| 136 """Get a value from SàT configuration | |
| 137 | |
| 138 A list is expected with either "name" of a config parameter, a one or more of | |
| 139 those parameters: | |
| 140 - section | |
| 141 - name | |
| 142 - default value | |
| 143 - filter | |
| 144 filter can be: | |
| 145 - "first": get the first item of the value | |
| 146 """ | |
| 147 config_data = loader.construct_sequence(node) | |
| 148 if len(config_data) == 1: | |
| 149 section, name, default, filter_ = "", config_data[0], None, None | |
| 150 if len(config_data) == 2: | |
| 151 (section, name), default, filter_ = config_data, None, None | |
| 152 elif len(config_data) == 3: | |
| 153 (section, name, default), filter_ = config_data, None | |
| 154 elif len(config_data) == 4: | |
| 155 section, name, default, filter_ = config_data | |
| 156 else: | |
| 157 raise ValueError( | |
| 158 f"invalid !sat_conf value ({config_data!r}), a list of 1 to 4 items is " | |
| 159 "expected" | |
| 160 ) | |
| 161 | |
| 162 value = self.host.memory.getConfig(section, name, default) | |
|
3417
bfe9ebd253a7
plugin app manager: fixed `public_url` value check
Goffi <goffi@goffi.org>
parents:
3372
diff
changeset
|
163 if name == "public_url" and (not value or not value.startswith('http')): |
| 3372 | 164 if not value: |
| 165 log.warning(_( | |
| 166 'No value found for "public_url", using "https://example.org" for ' | |
| 167 'now, please set the proper value in sat.conf')) | |
| 168 else: | |
| 169 log.warning(_( | |
| 170 'invalid value for "public_url" ({value}), it must start with ' | |
| 171 '"http", ignoring it and using "https://example.org" instead') | |
| 172 .format(value=value)) | |
| 173 value = "https://example.org" | |
| 174 | |
| 175 if filter_ is None: | |
| 176 pass | |
| 177 elif filter_ == 'first': | |
| 178 value = value[0] | |
| 179 else: | |
| 180 raise ValueError(f"unmanaged filter: {filter_}") | |
| 181 | |
| 182 return value | |
| 183 | |
| 184 def _sat_generate_pwd_constr(self, loader, node): | |
| 185 alphabet = string.ascii_letters + string.digits | |
| 186 return ''.join(secrets.choice(alphabet) for i in range(30)) | |
| 187 | |
| 188 def _sat_param_constr(self, loader, node): | |
| 189 """Get a parameter specified when starting the application | |
| 190 | |
| 191 The value can be either the name of the parameter to get, or a list as | |
| 192 [name, default_value] | |
| 193 """ | |
| 194 try: | |
| 195 name, default = loader.construct_sequence(node) | |
| 196 except ConstructorError: | |
| 197 name, default = loader.construct_scalar(node), None | |
| 198 return self._params.get(name, default) | |
| 199 | |
| 200 def register(self, manager): | |
| 201 name = manager.name | |
| 202 if name in self._managers: | |
| 203 raise exceptions.ConflictError( | |
| 204 f"There is already a manager with the name {name}") | |
| 205 self._managers[manager.name] = manager | |
| 206 if hasattr(manager, "discover_path"): | |
| 207 self.discover(manager.discover_path, manager) | |
| 208 | |
| 209 def getManager(self, app_data: dict) -> object: | |
| 210 """Get manager instance needed for this app | |
| 211 | |
| 212 @raise exceptions.DataError: something is wrong with the type | |
| 213 @raise exceptions.NotFound: manager is not registered | |
| 214 """ | |
| 215 try: | |
| 216 app_type = app_data["type"] | |
| 217 except KeyError: | |
| 218 raise exceptions.DataError( | |
| 219 "app file doesn't have the mandatory \"type\" key" | |
| 220 ) | |
| 221 if not isinstance(app_type, str): | |
| 222 raise exceptions.DataError( | |
| 223 f"invalid app data type: {app_type!r}" | |
| 224 ) | |
| 225 app_type = app_type.strip() | |
| 226 try: | |
| 227 return self._managers[app_type] | |
| 228 except KeyError: | |
| 229 raise exceptions.NotFound( | |
| 230 f"No manager found to manage app of type {app_type!r}") | |
| 231 | |
| 232 def getAppData( | |
| 233 self, | |
| 234 id_type: Optional[str], | |
| 235 identifier: str | |
| 236 ) -> dict: | |
| 237 """Retrieve instance's app_data from identifier | |
| 238 | |
| 239 @param id_type: type of the identifier, can be: | |
| 240 - "name": identifier is a canonical application name | |
| 241 the first found instance of this application is returned | |
| 242 - "instance": identifier is an instance id | |
| 243 @param identifier: identifier according to id_type | |
| 244 @return: instance application data | |
| 245 @raise exceptions.NotFound: no instance with this id can be found | |
| 246 @raise ValueError: id_type is invalid | |
| 247 """ | |
| 248 if not id_type: | |
| 249 id_type = 'name' | |
| 250 if id_type == 'name': | |
| 251 identifier = identifier.lower().strip() | |
| 252 try: | |
| 253 return next(iter(self._started[identifier])) | |
| 254 except (KeyError, StopIteration): | |
| 255 raise exceptions.NotFound( | |
| 256 f"No instance of {identifier!r} is currently running" | |
| 257 ) | |
| 258 elif id_type == 'instance': | |
| 259 instance_id = identifier | |
| 260 try: | |
| 261 return self._instances[instance_id] | |
| 262 except KeyError: | |
| 263 raise exceptions.NotFound( | |
| 264 f"There is no application instance running with id {instance_id!r}" | |
| 265 ) | |
| 266 else: | |
| 267 raise ValueError(f"invalid id_type: {id_type!r}") | |
| 268 | |
| 269 def discover( | |
| 270 self, | |
| 271 dir_path: Path, | |
| 272 manager: Optional = None | |
| 273 ) -> None: | |
| 274 for file_path in dir_path.glob(f"{APP_FILE_PREFIX}*.yaml"): | |
| 275 if manager is None: | |
| 276 try: | |
| 277 app_data = self.parse(file_path) | |
| 278 manager = self.getManager(app_data) | |
| 279 except (exceptions.DataError, exceptions.NotFound) as e: | |
| 280 log.warning( | |
| 281 f"Can't parse {file_path}, skipping: {e}") | |
| 282 app_name = file_path.stem[len(APP_FILE_PREFIX):].strip().lower() | |
| 283 if not app_name: | |
| 284 log.warning( | |
| 285 f"invalid app file name at {file_path}") | |
| 286 continue | |
| 287 app_dict = self._apps.setdefault(app_name, {}) | |
| 288 manager_set = app_dict.setdefault(manager, set()) | |
| 289 manager_set.add(file_path) | |
| 290 log.debug( | |
| 291 f"{app_name!r} {manager.name} application found" | |
| 292 ) | |
| 293 | |
| 294 def parse(self, file_path: Path, params: Optional[dict] = None) -> dict: | |
| 295 """Parse SàT application file | |
| 296 | |
| 297 @param params: parameters for running this instance | |
| 298 @raise exceptions.DataError: something is wrong in the file | |
| 299 """ | |
| 300 if params is None: | |
| 301 params = {} | |
| 302 with file_path.open() as f: | |
| 303 # we set parameters to be used only with this instance | |
| 304 # no async method must used between this assignation and `load` | |
| 305 self._params = params | |
| 306 app_data = self.load(f) | |
| 307 self._params = None | |
| 308 if "name" not in app_data: | |
| 309 # note that we don't use lower() here as we want human readable name and | |
| 310 # uppercase may be set on purpose | |
| 311 app_data['name'] = file_path.stem[len(APP_FILE_PREFIX):].strip() | |
| 312 single_instance = app_data.setdefault("single_instance", True) | |
| 313 if not isinstance(single_instance, bool): | |
| 314 raise ValueError( | |
| 315 f'"single_instance" must be a boolean, but it is {type(single_instance)}' | |
| 316 ) | |
| 317 return app_data | |
| 318 | |
| 319 def list_applications(self, filters: Optional[List[str]]) -> List[str]: | |
| 320 """List available application | |
| 321 | |
| 322 @param filters: only show applications matching those filters. | |
| 323 using None will list all known applications | |
| 324 a filter can be: | |
| 325 - available: applications available locally | |
| 326 - running: only show launched applications | |
| 327 """ | |
| 328 if not filters: | |
| 329 return list(self.apps) | |
| 330 found = set() | |
| 331 for filter_ in filters: | |
| 332 if filter_ == "available": | |
| 333 found.update(self._apps) | |
| 334 elif filter_ == "running": | |
| 335 found.update(self._started) | |
| 336 else: | |
| 337 raise ValueError(f"Unknown filter: {filter_}") | |
| 338 return list(found) | |
| 339 | |
| 340 def _start(self, app_name, extra): | |
| 341 extra = data_format.deserialise(extra) | |
| 342 return defer.ensureDeferred(self.start(str(app_name), extra)) | |
| 343 | |
| 344 async def start( | |
| 345 self, | |
| 346 app_name: str, | |
| 347 extra: Optional[dict] = None, | |
| 348 ) -> None: | |
| 349 # FIXME: for now we use the first app manager available for the requested app_name | |
| 350 # TODO: implement running multiple instance of the same app if some metadata | |
| 351 # to be defined in app_data allows explicitly it. | |
| 352 app_name = app_name.lower().strip() | |
| 353 try: | |
| 354 app_file_path = next(iter(next(iter(self._apps[app_name].values())))) | |
| 355 except KeyError: | |
| 356 raise exceptions.NotFound( | |
| 357 f"No application found with the name {app_name!r}" | |
| 358 ) | |
| 359 started_data = self._started.setdefault(app_name, []) | |
| 360 app_data = self.parse(app_file_path, extra) | |
| 361 app_data['_file_path'] = app_file_path | |
| 362 app_data['_name_canonical'] = app_name | |
| 363 single_instance = app_data['single_instance'] | |
| 364 if single_instance: | |
| 365 if started_data: | |
| 366 log.info(f"{app_name!r} is already started") | |
| 367 return | |
| 368 else: | |
| 369 cache_path = self.host.memory.getCachePath( | |
| 370 PLUGIN_INFO[C.PI_IMPORT_NAME], app_name | |
| 371 ) | |
| 372 cache_path.mkdir(0o700, parents=True, exist_ok=True) | |
| 373 app_data['_instance_dir_path'] = cache_path | |
| 374 else: | |
| 375 dest_dir_obj = tempfile.TemporaryDirectory(prefix="sat_app_") | |
| 376 app_data['_instance_dir_obj'] = dest_dir_obj | |
| 377 app_data['_instance_dir_path'] = Path(dest_dir_obj.name) | |
| 378 instance_id = app_data['_instance_id'] = shortuuid.uuid() | |
| 379 manager = self.getManager(app_data) | |
| 380 app_data['_manager'] = manager | |
| 381 started_data.append(app_data) | |
| 382 self._instances[instance_id] = app_data | |
| 383 | |
| 384 try: | |
| 385 start = manager.start | |
| 386 except AttributeError: | |
| 387 raise exceptions.InternalError( | |
| 388 f"{manager.name} doesn't have the mandatory \"start\" method" | |
| 389 ) | |
| 390 else: | |
| 391 await start(app_data) | |
| 392 log.info(f"{app_name!r} started") | |
| 393 | |
| 394 def _stop(self, identifier, id_type, extra): | |
| 395 extra = data_format.deserialise(extra) | |
| 396 return defer.ensureDeferred( | |
| 397 self.stop(str(identifier), str(id_type) or None, extra)) | |
| 398 | |
| 399 async def stop( | |
| 400 self, | |
| 401 identifier: str, | |
| 402 id_type: Optional[str] = None, | |
| 403 extra: Optional[dict] = None, | |
| 404 ) -> None: | |
| 405 if extra is None: | |
| 406 extra = {} | |
| 407 | |
| 408 app_data = self.getAppData(id_type, identifier) | |
| 409 | |
| 410 log.info(f"stopping {app_data['name']!r}") | |
| 411 | |
| 412 app_name = app_data['_name_canonical'] | |
| 413 instance_id = app_data['_instance_id'] | |
| 414 manager = app_data['_manager'] | |
| 415 | |
| 416 try: | |
| 417 stop = manager.stop | |
| 418 except AttributeError: | |
| 419 raise exceptions.InternalError( | |
| 420 f"{manager.name} doesn't have the mandatory \"stop\" method" | |
| 421 ) | |
| 422 else: | |
| 423 try: | |
| 424 await stop(app_data) | |
| 425 except Exception as e: | |
| 426 log.warning( | |
| 427 f"Instance {instance_id} of application {app_name} can't be stopped " | |
| 428 f"properly: {e}" | |
| 429 ) | |
| 430 return | |
| 431 | |
| 432 try: | |
| 433 del self._instances[instance_id] | |
| 434 except KeyError: | |
| 435 log.error( | |
| 436 f"INTERNAL ERROR: {instance_id!r} is not present in self._instances") | |
| 437 | |
| 438 try: | |
| 439 self._started[app_name].remove(app_data) | |
| 440 except ValueError: | |
| 441 log.error( | |
| 442 "INTERNAL ERROR: there is no app data in self._started with id " | |
| 443 f"{instance_id!r}" | |
| 444 ) | |
| 445 | |
| 446 log.info(f"{app_name!r} stopped") | |
| 447 | |
| 448 def _getExposed(self, identifier, id_type, extra): | |
| 449 extra = data_format.deserialise(extra) | |
| 450 d = defer.ensureDeferred(self.getExposed(identifier, id_type, extra)) | |
| 451 d.addCallback(lambda d: data_format.serialise(d)) | |
| 452 return d | |
| 453 | |
| 454 def getValueFromPath(self, app_data: dict, path: List[str]) -> any: | |
| 455 """Retrieve a value set in the data from it path | |
| 456 | |
| 457 @param path: list of key to use in app_data to retrieve the value | |
| 458 @return: found value | |
| 459 @raise NotFound: the value can't be found | |
| 460 """ | |
| 461 | |
| 462 async def getExposed( | |
| 463 self, | |
| 464 identifier: str, | |
| 465 id_type: Optional[str] = None, | |
| 466 extra: Optional[dict] = None, | |
| 467 ) -> dict: | |
| 468 """Get data exposed by the application | |
| 469 | |
| 470 The manager's "computeExpose" method will be called if it exists. It can be used | |
| 471 to handle manager specific conventions. | |
| 472 """ | |
| 473 app_data = self.getAppData(id_type, identifier) | |
| 474 if app_data.get('_exposed_computed', False): | |
| 475 return app_data['expose'] | |
| 476 if extra is None: | |
| 477 extra = {} | |
| 478 expose = app_data.setdefault("expose", {}) | |
| 479 if "passwords" in expose: | |
| 480 passwords = expose['passwords'] | |
| 481 for name, value in list(passwords.items()): | |
| 482 if isinstance(value, list): | |
| 483 # if we have a list, is the sequence of keys leading to the value | |
| 484 # to expose. We use "reduce" to retrieve the desired value | |
| 485 try: | |
| 486 passwords[name] = reduce(lambda l, k: l[k], value, app_data) | |
| 487 except Exception as e: | |
| 488 log.warning( | |
| 489 f"Can't retrieve exposed value for password {name!r}: {e}") | |
| 490 del passwords[name] | |
| 491 | |
| 492 url_prefix = expose.get("url_prefix") | |
| 493 if isinstance(url_prefix, list): | |
| 494 try: | |
| 495 expose["url_prefix"] = reduce(lambda l, k: l[k], url_prefix, app_data) | |
| 496 except Exception as e: | |
| 497 log.warning( | |
| 498 f"Can't retrieve exposed value for url_prefix: {e}") | |
| 499 del expose["url_prefix"] | |
| 500 | |
| 501 try: | |
| 502 computeExpose = app_data['_manager'].computeExpose | |
| 503 except AttributeError: | |
| 504 pass | |
| 505 else: | |
| 506 await computeExpose(app_data) | |
| 507 | |
| 508 app_data['_exposed_computed'] = True | |
| 509 return expose | |
| 510 | |
| 511 async def _doPrepare( | |
| 512 self, | |
| 513 app_data: dict, | |
| 514 ) -> None: | |
| 515 name = app_data['name'] | |
| 516 dest_path = app_data['_instance_dir_path'] | |
| 517 if next(dest_path.iterdir(), None) != None: | |
| 518 log.debug(f"There is already a prepared dir at {dest_path}, nothing to do") | |
| 519 return | |
| 520 try: | |
| 521 prepare = app_data['prepare'].copy() | |
| 522 except KeyError: | |
| 523 prepare = {} | |
| 524 | |
| 525 if not prepare: | |
| 526 log.debug("Nothing to prepare for {name!r}") | |
| 527 return | |
| 528 | |
| 529 for action, value in list(prepare.items()): | |
| 530 log.debug(f"[{name}] [prepare] running {action!r} action") | |
| 531 if action == "git": | |
| 532 try: | |
| 533 git_path = which('git')[0] | |
| 534 except IndexError: | |
| 535 raise exceptions.NotFound( | |
| 536 "Can't find \"git\" executable, {name} can't be started without it" | |
| 537 ) | |
| 538 await async_process.run(git_path, "clone", value, str(dest_path)) | |
| 539 log.debug(f"{value!r} git repository cloned at {dest_path}") | |
| 540 else: | |
| 541 raise NotImplementedError( | |
| 542 f"{action!r} is not managed, can't start {name}" | |
| 543 ) | |
| 544 del prepare[action] | |
| 545 | |
| 546 if prepare: | |
| 547 raise exceptions.InternalError('"prepare" should be empty') | |
| 548 | |
| 549 async def startCommon(self, app_data: dict) -> None: | |
| 550 """Method running common action when starting a manager | |
| 551 | |
| 552 It should be called by managers in "start" method. | |
| 553 """ | |
| 554 log.info(f"starting {app_data['name']!r}") | |
| 555 await self._doPrepare(app_data) |