Mercurial > libervia-backend

comparison libervia/backend/plugins/plugin_misc_app_manager/__init__.py @ 4247:4aa62767f501

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
plugin app manager: various improvements: - Generated password must now be named and are stored, so they are re-used on following restarts. Password size can now be specified. - New `not` filter for `!libervia_param` to inverse a boolean value. - Former `front_url` field has been renamed to `web_url_path` as it is the URL path used for web frontend. All Web frontend related field are prefixed with `web_`. - `front_url` is now used to specify a whole front URL (notably useful if an app uses its own domain). A list can be used to retrieve a key, like for `url_prefix`, and `https` scheme is added if no scheme is specified. - An abstract class is now used for App Managers. - Last application start time is stored in persistent data.
author Goffi <goffi@goffi.org>
date Fri, 31 May 2024 11:08:14 +0200
parents libervia/backend/plugins/plugin_misc_app_manager.py@c93b02000ae4
children 0d7bb4df2343
comparison
equal deleted inserted replaced
4246:5eb13251fd75 4247:4aa62767f501
1 #!/usr/bin/env python3
2
3 # Libervia plugin to manage external applications
4 # Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org)
5
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
10
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
15
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18
19 from functools import partial, reduce
20 from pathlib import Path
21 import secrets
22 import string
23 import tempfile
24 import time
25 from typing import Any, Callable, List, Optional
26 from urllib.parse import urlparse, urlunparse
27
28 import shortuuid
29 from twisted.internet import defer
30 from twisted.python.procutils import which
31 from yaml.constructor import ConstructorError
32
33 from libervia.backend.core import exceptions
34 from libervia.backend.core.constants import Const as C
35 from libervia.backend.core.i18n import _
36 from libervia.backend.core.log import getLogger
37 from libervia.backend.memory import persistent
38 from libervia.backend.plugins.plugin_misc_app_manager.models import AppManagerBackend
39 from libervia.backend.tools.common import data_format
40 from libervia.backend.tools.common import async_process
41
42 log = getLogger(__name__)
43
44 try:
45 import yaml
46 except ImportError:
47 raise exceptions.MissingModule(
48 'Missing module PyYAML, please download/install it. You can use '
49 '"pip install pyyaml"'
50 )
51
52 try:
53 from yaml import CLoader as Loader, CDumper as Dumper
54 except ImportError:
55 log.warning(
56 "Can't use LibYAML binding (is libyaml installed?), pure Python version will be "
57 "used, but it is slower"
58 )
59 from yaml import Loader, Dumper
60
61
62
63
64 PLUGIN_INFO = {
65 C.PI_NAME: "Applications Manager",
66 C.PI_IMPORT_NAME: "APP_MANAGER",
67 C.PI_TYPE: C.PLUG_TYPE_MISC,
68 C.PI_MODES: C.PLUG_MODE_BOTH,
69 C.PI_MAIN: "AppManager",
70 C.PI_HANDLER: "no",
71 C.PI_DESCRIPTION: _(
72 """Applications Manager
73
74 Manage external applications using packagers, OS virtualization/containers or other
75 software management tools.
76 """),
77 }
78
79 APP_FILE_PREFIX = "libervia_app_"
80
81
82 class AppManager:
83 load = partial(yaml.load, Loader=Loader)
84 dump = partial(yaml.dump, Dumper=Dumper)
85
86 def __init__(self, host):
87 log.info(_("plugin Applications Manager initialization"))
88 self.host = host
89 self._managers = {}
90 self._apps = {}
91 self._started = {}
92 # instance id to app data map
93 self._instances = {}
94
95
96 self.persistent_data = persistent.LazyPersistentBinaryDict("app_manager")
97
98 host.bridge.add_method(
99 "applications_list",
100 ".plugin",
101 in_sign="as",
102 out_sign="as",
103 method=self.list_applications,
104 )
105 host.bridge.add_method(
106 "application_start",
107 ".plugin",
108 in_sign="ss",
109 out_sign="s",
110 method=self._start,
111 async_=True,
112 )
113 host.bridge.add_method(
114 "application_stop",
115 ".plugin",
116 in_sign="sss",
117 out_sign="",
118 method=self._stop,
119 async_=True,
120 )
121 host.bridge.add_method(
122 "application_exposed_get",
123 ".plugin",
124 in_sign="sss",
125 out_sign="s",
126 method=self._get_exposed,
127 async_=True,
128 )
129 # application has been started succeesfully,
130 # args: name, instance_id, extra
131 host.bridge.add_signal(
132 "application_started", ".plugin", signature="sss"
133 )
134 # application went wrong with the application
135 # args: name, instance_id, extra
136 host.bridge.add_signal(
137 "application_error", ".plugin", signature="sss"
138 )
139 yaml.add_constructor(
140 "!libervia_conf", self._libervia_conf_constr, Loader=Loader)
141 yaml.add_constructor(
142 "!libervia_generate_pwd", self._libervia_generate_pwd_constr, Loader=Loader)
143 yaml.add_constructor(
144 "!libervia_param", self._libervia_param_constr, Loader=Loader)
145
146 def unload(self):
147 log.debug("unloading applications manager")
148 for instances in self._started.values():
149 for instance in instances:
150 data = instance['data']
151 if not data['single_instance']:
152 log.debug(
153 f"cleaning temporary directory at {data['_instance_dir_path']}")
154 data['_instance_dir_obj'].cleanup()
155
156 def _libervia_conf_constr(self, loader, node) -> str:
157 """Get a value from Libervia configuration
158
159 A list is expected with either "name" of a config parameter, a one or more of
160 those parameters:
161 - section
162 - name
163 - default value
164 - filter
165 filter can be:
166 - "first": get the first item of the value
167 - "not": get the opposite value (to be used with booleans)
168 """
169 config_data = loader.construct_sequence(node)
170 if len(config_data) == 1:
171 section, name, default, filter_ = "", config_data[0], None, None
172 if len(config_data) == 2:
173 (section, name), default, filter_ = config_data, None, None
174 elif len(config_data) == 3:
175 (section, name, default), filter_ = config_data, None
176 elif len(config_data) == 4:
177 section, name, default, filter_ = config_data
178 else:
179 raise ValueError(
180 f"invalid !libervia_conf value ({config_data!r}), a list of 1 to 4 items "
181 "is expected"
182 )
183
184 value = self.host.memory.config_get(section, name, default)
185 # FIXME: "public_url" is used only here and doesn't take multi-sites into account
186 if name == "public_url" and (not value or value.startswith('http')):
187 if not value:
188 log.warning(_(
189 'No value found for "public_url", using "example.org" for '
190 'now, please set the proper value in libervia.conf'))
191 else:
192 log.warning(_(
193 'invalid value for "public_url" ({value}), it musts not start with '
194 'schema ("http"), ignoring it and using "example.org" '
195 'instead')
196 .format(value=value))
197 value = "example.org"
198
199 if filter_ is None:
200 pass
201 elif filter_ == 'first':
202 value = value[0]
203 elif filter_ == "not":
204 value = C.bool(value)
205 value = C.bool_const(not value).lower()
206 else:
207 raise ValueError(f"unmanaged filter: {filter_}")
208
209 return value
210
211 def _libervia_generate_pwd_constr(self, loader, node) -> str:
212 """Generate a password and store it in persistent data.
213
214 If the password has already been generated previously, it is reused.
215 Mapping arguments are:
216 ``name`` (str) **required**
217 Name of the password. Will be used to store it.
218 ``size`` (int)
219 Size of the password to generate. Default to 30
220 """
221 try:
222 kwargs = loader.construct_mapping(node)
223 name = kwargs["name"]
224 except (ConstructorError, KeyError):
225 raise ValueError(
226 '!libervia_generate_pwd map arguments is missing. At least "name" '
227 "must be specified."
228 )
229 pwd_data_key = f"pwd_{name}"
230 try:
231 key = self._app_persistent_data[pwd_data_key]
232 except KeyError:
233 alphabet = string.ascii_letters + string.digits
234 key_size = int(kwargs.get("size", 30))
235 key = ''.join(secrets.choice(alphabet) for __ in range(key_size))
236 self._app_persistent_data[pwd_data_key] = key
237 else:
238 log.debug(f"Re-using existing key for {name!r} password.")
239 return key
240
241 def _libervia_param_constr(self, loader, node) -> str:
242 """Get a parameter specified when starting the application
243
244 The value can be either the name of the parameter to get, or a list as
245 [name, default_value]
246 """
247 try:
248 name, default = loader.construct_sequence(node)
249 except ConstructorError:
250 name, default = loader.construct_scalar(node), None
251 assert self._params is not None
252 return self._params.get(name, default)
253
254 def register(self, manager):
255 name = manager.name
256 if name in self._managers:
257 raise exceptions.ConflictError(
258 f"There is already a manager with the name {name}")
259 self._managers[manager.name] = manager
260 if manager.discover_path is not None:
261 self.discover(manager.discover_path, manager)
262
263 def get_manager(self, app_data: dict) -> AppManagerBackend:
264 """Get manager instance needed for this app
265
266 @raise exceptions.DataError: something is wrong with the type
267 @raise exceptions.NotFound: manager is not registered
268 """
269 try:
270 app_type = app_data["type"]
271 except KeyError:
272 raise exceptions.DataError(
273 "app file doesn't have the mandatory \"type\" key"
274 )
275 if not isinstance(app_type, str):
276 raise exceptions.DataError(
277 f"invalid app data type: {app_type!r}"
278 )
279 app_type = app_type.strip()
280 try:
281 return self._managers[app_type]
282 except KeyError:
283 raise exceptions.NotFound(
284 f"No manager found to manage app of type {app_type!r}")
285
286 def get_app_data(
287 self,
288 id_type: Optional[str],
289 identifier: str
290 ) -> dict:
291 """Retrieve instance's app_data from identifier
292
293 @param id_type: type of the identifier, can be:
294 - "name": identifier is a canonical application name
295 the first found instance of this application is returned
296 - "instance": identifier is an instance id
297 @param identifier: identifier according to id_type
298 @return: instance application data
299 @raise exceptions.NotFound: no instance with this id can be found
300 @raise ValueError: id_type is invalid
301 """
302 if not id_type:
303 id_type = 'name'
304 if id_type == 'name':
305 identifier = identifier.lower().strip()
306 try:
307 return next(iter(self._started[identifier]))
308 except (KeyError, StopIteration):
309 raise exceptions.NotFound(
310 f"No instance of {identifier!r} is currently running"
311 )
312 elif id_type == 'instance':
313 instance_id = identifier
314 try:
315 return self._instances[instance_id]
316 except KeyError:
317 raise exceptions.NotFound(
318 f"There is no application instance running with id {instance_id!r}"
319 )
320 else:
321 raise ValueError(f"invalid id_type: {id_type!r}")
322
323 def discover(
324 self,
325 dir_path: Path,
326 manager: AppManagerBackend|None = None
327 ) -> None:
328 """Search for app configuration file.
329
330 App configuration files must start with [APP_FILE_PREFIX] and have a ``.yaml``
331 extension.
332 """
333 for file_path in dir_path.glob(f"{APP_FILE_PREFIX}*.yaml"):
334 if manager is None:
335 try:
336 app_data = self.parse(file_path)
337 manager = self.get_manager(app_data)
338 except (exceptions.DataError, exceptions.NotFound) as e:
339 log.warning(
340 f"Can't parse {file_path}, skipping: {e}")
341 continue
342 app_name = file_path.stem[len(APP_FILE_PREFIX):].strip().lower()
343 if not app_name:
344 log.warning(f"invalid app file name at {file_path}")
345 continue
346 app_dict = self._apps.setdefault(app_name, {})
347 manager_set = app_dict.setdefault(manager, set())
348 manager_set.add(file_path)
349 log.debug(
350 f"{app_name!r} {manager.name} application found"
351 )
352
353 def parse(self, file_path: Path, params: Optional[dict] = None) -> dict:
354 """Parse Libervia application file
355
356 @param params: parameters for running this instance
357 @raise exceptions.DataError: something is wrong in the file
358 """
359 if params is None:
360 params = {}
361 with file_path.open() as f:
362 # we set parameters to be used only with this instance
363 # no async method must used between this assignation and `load`
364 self._params = params
365 app_data = self.load(f)
366 self._params = None
367 if "name" not in app_data:
368 # note that we don't use lower() here as we want human readable name and
369 # uppercase may be set on purpose
370 app_data['name'] = file_path.stem[len(APP_FILE_PREFIX):].strip()
371 single_instance = app_data.setdefault("single_instance", True)
372 if not isinstance(single_instance, bool):
373 raise ValueError(
374 f'"single_instance" must be a boolean, but it is {type(single_instance)}'
375 )
376 return app_data
377
378 def list_applications(self, filters: Optional[List[str]]) -> List[str]:
379 """List available application
380
381 @param filters: only show applications matching those filters.
382 using None will list all known applications
383 a filter can be:
384 - available: applications available locally
385 - running: only show launched applications
386 """
387 if not filters:
388 return list(self.apps)
389 found = set()
390 for filter_ in filters:
391 if filter_ == "available":
392 found.update(self._apps)
393 elif filter_ == "running":
394 found.update(self._started)
395 else:
396 raise ValueError(f"Unknown filter: {filter_}")
397 return list(found)
398
399 def _start(self, app_name, extra):
400 extra = data_format.deserialise(extra)
401 d = defer.ensureDeferred(self.start(str(app_name), extra))
402 d.addCallback(data_format.serialise)
403 return d
404
405 async def start(
406 self,
407 app_name: str,
408 extra: Optional[dict] = None,
409 ) -> dict:
410 """Start an application
411
412 @param app_name: name of the application to start
413 @param extra: extra parameters
414 @return: data with following keys:
415 - name (str): canonical application name
416 - instance (str): instance ID
417 - started (bool): True if the application is already started
418 if False, the "application_started" signal should be used to get notificed
419 when the application is actually started
420 - expose (dict): exposed data as given by [self.get_exposed]
421 exposed data which need to be computed are NOT returned, they will
422 available when the app will be fully started, throught the
423 [self.get_exposed] method.
424 """
425 # FIXME: for now we use the first app manager available for the requested app_name
426 # TODO: implement running multiple instance of the same app if some metadata
427 # to be defined in app_data allows explicitly it.
428 app_name = app_name.lower().strip()
429 try:
430 app_file_path = next(iter(next(iter(self._apps[app_name].values()))))
431 except KeyError:
432 raise exceptions.NotFound(
433 f"No application found with the name {app_name!r}"
434 )
435 log.info(f"starting {app_name!r}")
436 self._app_persistent_data = await self.persistent_data.get(app_name) or {}
437 self._app_persistent_data["last_started"] = time.time()
438 started_data = self._started.setdefault(app_name, [])
439 app_data = self.parse(app_file_path, extra)
440 await self.persistent_data.aset(app_name, self._app_persistent_data)
441 app_data["_started"] = False
442 app_data['_file_path'] = app_file_path
443 app_data['_name_canonical'] = app_name
444 single_instance = app_data['single_instance']
445 ret_data = {
446 "name": app_name,
447 "started": False
448 }
449 if single_instance:
450 if started_data:
451 instance_data = started_data[0]
452 instance_id = instance_data["_instance_id"]
453 ret_data["instance"] = instance_id
454 ret_data["started"] = instance_data["_started"]
455 ret_data["expose"] = await self.get_exposed(
456 instance_id, "instance", {"skip_compute": True}
457 )
458 log.info(f"{app_name!r} is already started or being started")
459 return ret_data
460 else:
461 cache_path = self.host.memory.get_cache_path(
462 PLUGIN_INFO[C.PI_IMPORT_NAME], app_name
463 )
464 cache_path.mkdir(0o700, parents=True, exist_ok=True)
465 app_data['_instance_dir_path'] = cache_path
466 else:
467 dest_dir_obj = tempfile.TemporaryDirectory(prefix="libervia_app_")
468 app_data['_instance_dir_obj'] = dest_dir_obj
469 app_data['_instance_dir_path'] = Path(dest_dir_obj.name)
470 instance_id = ret_data["instance"] = app_data['_instance_id'] = shortuuid.uuid()
471 manager = self.get_manager(app_data)
472 app_data['_manager'] = manager
473 started_data.append(app_data)
474 self._instances[instance_id] = app_data
475 # we retrieve exposed data such as url_prefix which can be useful computed exposed
476 # data must wait for the app to be started, so we skip them for now
477 ret_data["expose"] = await self.get_exposed(
478 instance_id, "instance", {"skip_compute": True}
479 )
480
481 try:
482 start = manager.start
483 except AttributeError:
484 raise exceptions.InternalError(
485 f"{manager.name} doesn't have the mandatory \"start\" method"
486 )
487 else:
488 defer.ensureDeferred(self.start_app(start, app_data))
489 return ret_data
490
491 async def start_app(self, start_cb: Callable, app_data: dict) -> None:
492 app_name = app_data["_name_canonical"]
493 instance_id = app_data["_instance_id"]
494 try:
495 await start_cb(app_data)
496 except Exception as e:
497 log.exception(f"Can't start libervia app {app_name!r}")
498 self.host.bridge.application_error(
499 app_name,
500 instance_id,
501 data_format.serialise(
502 {
503 "class": str(type(e)),
504 "msg": str(e)
505 }
506 ))
507 else:
508 app_data["_started"] = True
509 self.host.bridge.application_started(app_name, instance_id, "")
510 log.info(f"{app_name!r} started")
511
512 def _stop(self, identifier, id_type, extra):
513 extra = data_format.deserialise(extra)
514 return defer.ensureDeferred(
515 self.stop(str(identifier), str(id_type) or None, extra))
516
517 async def stop(
518 self,
519 identifier: str,
520 id_type: Optional[str] = None,
521 extra: Optional[dict] = None,
522 ) -> None:
523 if extra is None:
524 extra = {}
525
526 app_data = self.get_app_data(id_type, identifier)
527
528 log.info(f"stopping {app_data['name']!r}")
529
530 app_name = app_data['_name_canonical']
531 instance_id = app_data['_instance_id']
532 manager = app_data['_manager']
533
534 try:
535 stop = manager.stop
536 except AttributeError:
537 raise exceptions.InternalError(
538 f"{manager.name} doesn't have the mandatory \"stop\" method"
539 )
540 else:
541 try:
542 await stop(app_data)
543 except Exception as e:
544 log.warning(
545 f"Instance {instance_id} of application {app_name} can't be stopped "
546 f"properly: {e}"
547 )
548 return
549
550 try:
551 del self._instances[instance_id]
552 except KeyError:
553 log.error(
554 f"INTERNAL ERROR: {instance_id!r} is not present in self._instances")
555
556 try:
557 self._started[app_name].remove(app_data)
558 except ValueError:
559 log.error(
560 "INTERNAL ERROR: there is no app data in self._started with id "
561 f"{instance_id!r}"
562 )
563
564 log.info(f"{app_name!r} stopped")
565
566 def _get_exposed(self, identifier, id_type, extra):
567 extra = data_format.deserialise(extra)
568 d = defer.ensureDeferred(self.get_exposed(identifier, id_type, extra))
569 d.addCallback(lambda d: data_format.serialise(d))
570 return d
571
572 def get_app_data_value(self, path: list[str], data: dict[str, Any]) -> Any:
573 """Return a value from app data from its path.
574
575 @param path: sequence of keys to get to retrieve the value
576 @return: requested value
577 @raise KeyError: Can't find the requested value.
578 """
579 return reduce(lambda l, k: l[k], path, data)
580
581 async def get_exposed(
582 self,
583 identifier: str,
584 id_type: Optional[str] = None,
585 extra: Optional[dict] = None,
586 ) -> dict:
587 """Get data exposed by the application
588
589 The manager's "compute_expose" method will be called if it exists. It can be used
590 to handle manager specific conventions.
591 """
592 app_data = self.get_app_data(id_type, identifier)
593 if app_data.get('_exposed_computed', False):
594 return app_data['expose']
595 if extra is None:
596 extra = {}
597 expose = app_data.setdefault("expose", {})
598 if "passwords" in expose:
599 passwords = expose['passwords']
600 for name, value in list(passwords.items()):
601 if isinstance(value, list):
602 # if we have a list, is the sequence of keys leading to the value
603 # to expose.
604 try:
605 passwords[name] = self.get_app_data_value(value, app_data)
606 except KeyError:
607 log.warning(
608 f"Can't retrieve exposed value for password {name!r}: {e}")
609 del passwords[name]
610
611 for key in ("url_prefix", "front_url"):
612 value = expose.get(key)
613 if isinstance(value, list):
614 try:
615 expose[key] = self.get_app_data_value(value, app_data)
616 except KeyError:
617 log.warning(
618 f"Can't retrieve exposed value for {key!r} at {value}"
619 )
620 del expose[key]
621
622 front_url = expose.get("front_url")
623 if front_url:
624 # we want to be sure that a scheme is defined, defaulting to ``https``
625 parsed_url = urlparse(front_url)
626 if not parsed_url.scheme:
627 if not parsed_url.netloc:
628 path_elt = parsed_url.path.split("/", 1)
629 parsed_url = parsed_url._replace(
630 netloc=path_elt[0],
631 path=f"/{path_elt[1]}" if len(path_elt) > 1 else ""
632 )
633 parsed_url = parsed_url._replace(scheme='https')
634 expose["front_url"] = urlunparse(parsed_url)
635
636 if extra.get("skip_compute", False):
637 return expose
638
639 try:
640 compute_expose = app_data['_manager'].compute_expose
641 except AttributeError:
642 pass
643 else:
644 await compute_expose(app_data)
645
646 app_data['_exposed_computed'] = True
647 return expose
648
649 async def _do_prepare(
650 self,
651 app_data: dict,
652 ) -> None:
653 name = app_data['name']
654 dest_path = app_data['_instance_dir_path']
655 if next(dest_path.iterdir(), None) != None:
656 log.debug(f"There is already a prepared dir at {dest_path}, nothing to do")
657 return
658 try:
659 prepare = app_data['prepare'].copy()
660 except KeyError:
661 prepare = {}
662
663 if not prepare:
664 log.debug(f"Nothing to prepare for {name!r}")
665 return
666
667 for action, value in list(prepare.items()):
668 log.debug(f"[{name}] [prepare] running {action!r} action")
669 if action == "git":
670 try:
671 git_path = which('git')[0]
672 except IndexError:
673 raise exceptions.NotFound(
674 "Can't find \"git\" executable, {name} can't be started without it"
675 )
676 await async_process.run(git_path, "clone", value, str(dest_path))
677 log.debug(f"{value!r} git repository cloned at {dest_path}")
678 else:
679 raise NotImplementedError(
680 f"{action!r} is not managed, can't start {name}"
681 )
682 del prepare[action]
683
684 if prepare:
685 raise exceptions.InternalError('"prepare" should be empty')
686
687 async def _do_create_files(
688 self,
689 app_data: dict,
690 ) -> None:
691 dest_path = app_data['_instance_dir_path']
692 files = app_data.get('files')
693 if not files:
694 return
695 if not isinstance(files, dict):
696 raise ValueError('"files" must be a dictionary')
697 for filename, data in files.items():
698 path = dest_path / filename
699 if path.is_file():
700 log.info(f"{path} already exists, skipping")
701 with path.open("w") as f:
702 f.write(data.get("content", ""))
703 log.debug(f"{path} created")
704
705 async def start_common(self, app_data: dict) -> None:
706 """Method running common action when starting a manager
707
708 It should be called by managers in "start" method.
709 """
710 await self._do_prepare(app_data)
711 await self._do_create_files(app_data)