Mercurial > libervia-backend

comparison libervia/backend/plugins/plugin_misc_app_manager.py @ 4071:4b842c1fb686

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
refactoring: renamed `sat` package to `libervia.backend`
author Goffi <goffi@goffi.org>
date Fri, 02 Jun 2023 11:49:51 +0200
parents sat/plugins/plugin_misc_app_manager.py@524856bd7b19
children c93b02000ae4
comparison
equal deleted inserted replaced
4070:d10748475025 4071:4b842c1fb686
1 #!/usr/bin/env python3
2
3 # Libervia plugin to manage external applications
4 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
5
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
10
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
15
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18
19 from pathlib import Path
20 from typing import Optional, List, Callable
21 from functools import partial, reduce
22 import tempfile
23 import secrets
24 import string
25 import shortuuid
26 from twisted.internet import defer
27 from twisted.python.procutils import which
28 from libervia.backend.core.i18n import _
29 from libervia.backend.core import exceptions
30 from libervia.backend.core.constants import Const as C
31 from libervia.backend.core.log import getLogger
32 from libervia.backend.tools.common import data_format
33 from libervia.backend.tools.common import async_process
34
35 log = getLogger(__name__)
36
37 try:
38 import yaml
39 except ImportError:
40 raise exceptions.MissingModule(
41 'Missing module PyYAML, please download/install it. You can use '
42 '"pip install pyyaml"'
43 )
44
45 try:
46 from yaml import CLoader as Loader, CDumper as Dumper
47 except ImportError:
48 log.warning(
49 "Can't use LibYAML binding (is libyaml installed?), pure Python version will be "
50 "used, but it is slower"
51 )
52 from yaml import Loader, Dumper
53
54 from yaml.constructor import ConstructorError
55
56
57 PLUGIN_INFO = {
58 C.PI_NAME: "Applications Manager",
59 C.PI_IMPORT_NAME: "APP_MANAGER",
60 C.PI_TYPE: C.PLUG_TYPE_MISC,
61 C.PI_MODES: C.PLUG_MODE_BOTH,
62 C.PI_MAIN: "AppManager",
63 C.PI_HANDLER: "no",
64 C.PI_DESCRIPTION: _(
65 """Applications Manager
66
67 Manage external applications using packagers, OS virtualization/containers or other
68 software management tools.
69 """),
70 }
71
72 APP_FILE_PREFIX = "sat_app_"
73
74
75 class AppManager:
76 load = partial(yaml.load, Loader=Loader)
77 dump = partial(yaml.dump, Dumper=Dumper)
78
79 def __init__(self, host):
80 log.info(_("plugin Applications Manager initialization"))
81 self.host = host
82 self._managers = {}
83 self._apps = {}
84 self._started = {}
85 # instance id to app data map
86 self._instances = {}
87 host.bridge.add_method(
88 "applications_list",
89 ".plugin",
90 in_sign="as",
91 out_sign="as",
92 method=self.list_applications,
93 )
94 host.bridge.add_method(
95 "application_start",
96 ".plugin",
97 in_sign="ss",
98 out_sign="s",
99 method=self._start,
100 async_=True,
101 )
102 host.bridge.add_method(
103 "application_stop",
104 ".plugin",
105 in_sign="sss",
106 out_sign="",
107 method=self._stop,
108 async_=True,
109 )
110 host.bridge.add_method(
111 "application_exposed_get",
112 ".plugin",
113 in_sign="sss",
114 out_sign="s",
115 method=self._get_exposed,
116 async_=True,
117 )
118 # application has been started succeesfully,
119 # args: name, instance_id, extra
120 host.bridge.add_signal(
121 "application_started", ".plugin", signature="sss"
122 )
123 # application went wrong with the application
124 # args: name, instance_id, extra
125 host.bridge.add_signal(
126 "application_error", ".plugin", signature="sss"
127 )
128 yaml.add_constructor(
129 "!sat_conf", self._sat_conf_constr, Loader=Loader)
130 yaml.add_constructor(
131 "!sat_generate_pwd", self._sat_generate_pwd_constr, Loader=Loader)
132 yaml.add_constructor(
133 "!sat_param", self._sat_param_constr, Loader=Loader)
134
135 def unload(self):
136 log.debug("unloading applications manager")
137 for instances in self._started.values():
138 for instance in instances:
139 data = instance['data']
140 if not data['single_instance']:
141 log.debug(
142 f"cleaning temporary directory at {data['_instance_dir_path']}")
143 data['_instance_dir_obj'].cleanup()
144
145 def _sat_conf_constr(self, loader, node):
146 """Get a value from Libervia configuration
147
148 A list is expected with either "name" of a config parameter, a one or more of
149 those parameters:
150 - section
151 - name
152 - default value
153 - filter
154 filter can be:
155 - "first": get the first item of the value
156 """
157 config_data = loader.construct_sequence(node)
158 if len(config_data) == 1:
159 section, name, default, filter_ = "", config_data[0], None, None
160 if len(config_data) == 2:
161 (section, name), default, filter_ = config_data, None, None
162 elif len(config_data) == 3:
163 (section, name, default), filter_ = config_data, None
164 elif len(config_data) == 4:
165 section, name, default, filter_ = config_data
166 else:
167 raise ValueError(
168 f"invalid !sat_conf value ({config_data!r}), a list of 1 to 4 items is "
169 "expected"
170 )
171
172 value = self.host.memory.config_get(section, name, default)
173 # FIXME: "public_url" is used only here and doesn't take multi-sites into account
174 if name == "public_url" and (not value or value.startswith('http')):
175 if not value:
176 log.warning(_(
177 'No value found for "public_url", using "example.org" for '
178 'now, please set the proper value in libervia.conf'))
179 else:
180 log.warning(_(
181 'invalid value for "public_url" ({value}), it musts not start with '
182 'schema ("http"), ignoring it and using "example.org" '
183 'instead')
184 .format(value=value))
185 value = "example.org"
186
187 if filter_ is None:
188 pass
189 elif filter_ == 'first':
190 value = value[0]
191 else:
192 raise ValueError(f"unmanaged filter: {filter_}")
193
194 return value
195
196 def _sat_generate_pwd_constr(self, loader, node):
197 alphabet = string.ascii_letters + string.digits
198 return ''.join(secrets.choice(alphabet) for i in range(30))
199
200 def _sat_param_constr(self, loader, node):
201 """Get a parameter specified when starting the application
202
203 The value can be either the name of the parameter to get, or a list as
204 [name, default_value]
205 """
206 try:
207 name, default = loader.construct_sequence(node)
208 except ConstructorError:
209 name, default = loader.construct_scalar(node), None
210 return self._params.get(name, default)
211
212 def register(self, manager):
213 name = manager.name
214 if name in self._managers:
215 raise exceptions.ConflictError(
216 f"There is already a manager with the name {name}")
217 self._managers[manager.name] = manager
218 if hasattr(manager, "discover_path"):
219 self.discover(manager.discover_path, manager)
220
221 def get_manager(self, app_data: dict) -> object:
222 """Get manager instance needed for this app
223
224 @raise exceptions.DataError: something is wrong with the type
225 @raise exceptions.NotFound: manager is not registered
226 """
227 try:
228 app_type = app_data["type"]
229 except KeyError:
230 raise exceptions.DataError(
231 "app file doesn't have the mandatory \"type\" key"
232 )
233 if not isinstance(app_type, str):
234 raise exceptions.DataError(
235 f"invalid app data type: {app_type!r}"
236 )
237 app_type = app_type.strip()
238 try:
239 return self._managers[app_type]
240 except KeyError:
241 raise exceptions.NotFound(
242 f"No manager found to manage app of type {app_type!r}")
243
244 def get_app_data(
245 self,
246 id_type: Optional[str],
247 identifier: str
248 ) -> dict:
249 """Retrieve instance's app_data from identifier
250
251 @param id_type: type of the identifier, can be:
252 - "name": identifier is a canonical application name
253 the first found instance of this application is returned
254 - "instance": identifier is an instance id
255 @param identifier: identifier according to id_type
256 @return: instance application data
257 @raise exceptions.NotFound: no instance with this id can be found
258 @raise ValueError: id_type is invalid
259 """
260 if not id_type:
261 id_type = 'name'
262 if id_type == 'name':
263 identifier = identifier.lower().strip()
264 try:
265 return next(iter(self._started[identifier]))
266 except (KeyError, StopIteration):
267 raise exceptions.NotFound(
268 f"No instance of {identifier!r} is currently running"
269 )
270 elif id_type == 'instance':
271 instance_id = identifier
272 try:
273 return self._instances[instance_id]
274 except KeyError:
275 raise exceptions.NotFound(
276 f"There is no application instance running with id {instance_id!r}"
277 )
278 else:
279 raise ValueError(f"invalid id_type: {id_type!r}")
280
281 def discover(
282 self,
283 dir_path: Path,
284 manager: Optional = None
285 ) -> None:
286 for file_path in dir_path.glob(f"{APP_FILE_PREFIX}*.yaml"):
287 if manager is None:
288 try:
289 app_data = self.parse(file_path)
290 manager = self.get_manager(app_data)
291 except (exceptions.DataError, exceptions.NotFound) as e:
292 log.warning(
293 f"Can't parse {file_path}, skipping: {e}")
294 app_name = file_path.stem[len(APP_FILE_PREFIX):].strip().lower()
295 if not app_name:
296 log.warning(
297 f"invalid app file name at {file_path}")
298 continue
299 app_dict = self._apps.setdefault(app_name, {})
300 manager_set = app_dict.setdefault(manager, set())
301 manager_set.add(file_path)
302 log.debug(
303 f"{app_name!r} {manager.name} application found"
304 )
305
306 def parse(self, file_path: Path, params: Optional[dict] = None) -> dict:
307 """Parse Libervia application file
308
309 @param params: parameters for running this instance
310 @raise exceptions.DataError: something is wrong in the file
311 """
312 if params is None:
313 params = {}
314 with file_path.open() as f:
315 # we set parameters to be used only with this instance
316 # no async method must used between this assignation and `load`
317 self._params = params
318 app_data = self.load(f)
319 self._params = None
320 if "name" not in app_data:
321 # note that we don't use lower() here as we want human readable name and
322 # uppercase may be set on purpose
323 app_data['name'] = file_path.stem[len(APP_FILE_PREFIX):].strip()
324 single_instance = app_data.setdefault("single_instance", True)
325 if not isinstance(single_instance, bool):
326 raise ValueError(
327 f'"single_instance" must be a boolean, but it is {type(single_instance)}'
328 )
329 return app_data
330
331 def list_applications(self, filters: Optional[List[str]]) -> List[str]:
332 """List available application
333
334 @param filters: only show applications matching those filters.
335 using None will list all known applications
336 a filter can be:
337 - available: applications available locally
338 - running: only show launched applications
339 """
340 if not filters:
341 return list(self.apps)
342 found = set()
343 for filter_ in filters:
344 if filter_ == "available":
345 found.update(self._apps)
346 elif filter_ == "running":
347 found.update(self._started)
348 else:
349 raise ValueError(f"Unknown filter: {filter_}")
350 return list(found)
351
352 def _start(self, app_name, extra):
353 extra = data_format.deserialise(extra)
354 d = defer.ensureDeferred(self.start(str(app_name), extra))
355 d.addCallback(data_format.serialise)
356 return d
357
358 async def start(
359 self,
360 app_name: str,
361 extra: Optional[dict] = None,
362 ) -> dict:
363 """Start an application
364
365 @param app_name: name of the application to start
366 @param extra: extra parameters
367 @return: data with following keys:
368 - name (str): canonical application name
369 - instance (str): instance ID
370 - started (bool): True if the application is already started
371 if False, the "application_started" signal should be used to get notificed
372 when the application is actually started
373 - expose (dict): exposed data as given by [self.get_exposed]
374 exposed data which need to be computed are NOT returned, they will
375 available when the app will be fully started, throught the
376 [self.get_exposed] method.
377 """
378 # FIXME: for now we use the first app manager available for the requested app_name
379 # TODO: implement running multiple instance of the same app if some metadata
380 # to be defined in app_data allows explicitly it.
381 app_name = app_name.lower().strip()
382 try:
383 app_file_path = next(iter(next(iter(self._apps[app_name].values()))))
384 except KeyError:
385 raise exceptions.NotFound(
386 f"No application found with the name {app_name!r}"
387 )
388 log.info(f"starting {app_name!r}")
389 started_data = self._started.setdefault(app_name, [])
390 app_data = self.parse(app_file_path, extra)
391 app_data["_started"] = False
392 app_data['_file_path'] = app_file_path
393 app_data['_name_canonical'] = app_name
394 single_instance = app_data['single_instance']
395 ret_data = {
396 "name": app_name,
397 "started": False
398 }
399 if single_instance:
400 if started_data:
401 instance_data = started_data[0]
402 instance_id = instance_data["_instance_id"]
403 ret_data["instance"] = instance_id
404 ret_data["started"] = instance_data["_started"]
405 ret_data["expose"] = await self.get_exposed(
406 instance_id, "instance", {"skip_compute": True}
407 )
408 log.info(f"{app_name!r} is already started or being started")
409 return ret_data
410 else:
411 cache_path = self.host.memory.get_cache_path(
412 PLUGIN_INFO[C.PI_IMPORT_NAME], app_name
413 )
414 cache_path.mkdir(0o700, parents=True, exist_ok=True)
415 app_data['_instance_dir_path'] = cache_path
416 else:
417 dest_dir_obj = tempfile.TemporaryDirectory(prefix="sat_app_")
418 app_data['_instance_dir_obj'] = dest_dir_obj
419 app_data['_instance_dir_path'] = Path(dest_dir_obj.name)
420 instance_id = ret_data["instance"] = app_data['_instance_id'] = shortuuid.uuid()
421 manager = self.get_manager(app_data)
422 app_data['_manager'] = manager
423 started_data.append(app_data)
424 self._instances[instance_id] = app_data
425 # we retrieve exposed data such as url_prefix which can be useful computed exposed
426 # data must wait for the app to be started, so we skip them for now
427 ret_data["expose"] = await self.get_exposed(
428 instance_id, "instance", {"skip_compute": True}
429 )
430
431 try:
432 start = manager.start
433 except AttributeError:
434 raise exceptions.InternalError(
435 f"{manager.name} doesn't have the mandatory \"start\" method"
436 )
437 else:
438 defer.ensureDeferred(self.start_app(start, app_data))
439 return ret_data
440
441 async def start_app(self, start_cb: Callable, app_data: dict) -> None:
442 app_name = app_data["_name_canonical"]
443 instance_id = app_data["_instance_id"]
444 try:
445 await start_cb(app_data)
446 except Exception as e:
447 log.exception(f"Can't start libervia app {app_name!r}")
448 self.host.bridge.application_error(
449 app_name,
450 instance_id,
451 data_format.serialise(
452 {
453 "class": str(type(e)),
454 "msg": str(e)
455 }
456 ))
457 else:
458 app_data["_started"] = True
459 self.host.bridge.application_started(app_name, instance_id, "")
460 log.info(f"{app_name!r} started")
461
462 def _stop(self, identifier, id_type, extra):
463 extra = data_format.deserialise(extra)
464 return defer.ensureDeferred(
465 self.stop(str(identifier), str(id_type) or None, extra))
466
467 async def stop(
468 self,
469 identifier: str,
470 id_type: Optional[str] = None,
471 extra: Optional[dict] = None,
472 ) -> None:
473 if extra is None:
474 extra = {}
475
476 app_data = self.get_app_data(id_type, identifier)
477
478 log.info(f"stopping {app_data['name']!r}")
479
480 app_name = app_data['_name_canonical']
481 instance_id = app_data['_instance_id']
482 manager = app_data['_manager']
483
484 try:
485 stop = manager.stop
486 except AttributeError:
487 raise exceptions.InternalError(
488 f"{manager.name} doesn't have the mandatory \"stop\" method"
489 )
490 else:
491 try:
492 await stop(app_data)
493 except Exception as e:
494 log.warning(
495 f"Instance {instance_id} of application {app_name} can't be stopped "
496 f"properly: {e}"
497 )
498 return
499
500 try:
501 del self._instances[instance_id]
502 except KeyError:
503 log.error(
504 f"INTERNAL ERROR: {instance_id!r} is not present in self._instances")
505
506 try:
507 self._started[app_name].remove(app_data)
508 except ValueError:
509 log.error(
510 "INTERNAL ERROR: there is no app data in self._started with id "
511 f"{instance_id!r}"
512 )
513
514 log.info(f"{app_name!r} stopped")
515
516 def _get_exposed(self, identifier, id_type, extra):
517 extra = data_format.deserialise(extra)
518 d = defer.ensureDeferred(self.get_exposed(identifier, id_type, extra))
519 d.addCallback(lambda d: data_format.serialise(d))
520 return d
521
522 async def get_exposed(
523 self,
524 identifier: str,
525 id_type: Optional[str] = None,
526 extra: Optional[dict] = None,
527 ) -> dict:
528 """Get data exposed by the application
529
530 The manager's "compute_expose" method will be called if it exists. It can be used
531 to handle manager specific conventions.
532 """
533 app_data = self.get_app_data(id_type, identifier)
534 if app_data.get('_exposed_computed', False):
535 return app_data['expose']
536 if extra is None:
537 extra = {}
538 expose = app_data.setdefault("expose", {})
539 if "passwords" in expose:
540 passwords = expose['passwords']
541 for name, value in list(passwords.items()):
542 if isinstance(value, list):
543 # if we have a list, is the sequence of keys leading to the value
544 # to expose. We use "reduce" to retrieve the desired value
545 try:
546 passwords[name] = reduce(lambda l, k: l[k], value, app_data)
547 except Exception as e:
548 log.warning(
549 f"Can't retrieve exposed value for password {name!r}: {e}")
550 del passwords[name]
551
552 url_prefix = expose.get("url_prefix")
553 if isinstance(url_prefix, list):
554 try:
555 expose["url_prefix"] = reduce(lambda l, k: l[k], url_prefix, app_data)
556 except Exception as e:
557 log.warning(
558 f"Can't retrieve exposed value for url_prefix: {e}")
559 del expose["url_prefix"]
560
561 if extra.get("skip_compute", False):
562 return expose
563
564 try:
565 compute_expose = app_data['_manager'].compute_expose
566 except AttributeError:
567 pass
568 else:
569 await compute_expose(app_data)
570
571 app_data['_exposed_computed'] = True
572 return expose
573
574 async def _do_prepare(
575 self,
576 app_data: dict,
577 ) -> None:
578 name = app_data['name']
579 dest_path = app_data['_instance_dir_path']
580 if next(dest_path.iterdir(), None) != None:
581 log.debug(f"There is already a prepared dir at {dest_path}, nothing to do")
582 return
583 try:
584 prepare = app_data['prepare'].copy()
585 except KeyError:
586 prepare = {}
587
588 if not prepare:
589 log.debug("Nothing to prepare for {name!r}")
590 return
591
592 for action, value in list(prepare.items()):
593 log.debug(f"[{name}] [prepare] running {action!r} action")
594 if action == "git":
595 try:
596 git_path = which('git')[0]
597 except IndexError:
598 raise exceptions.NotFound(
599 "Can't find \"git\" executable, {name} can't be started without it"
600 )
601 await async_process.run(git_path, "clone", value, str(dest_path))
602 log.debug(f"{value!r} git repository cloned at {dest_path}")
603 else:
604 raise NotImplementedError(
605 f"{action!r} is not managed, can't start {name}"
606 )
607 del prepare[action]
608
609 if prepare:
610 raise exceptions.InternalError('"prepare" should be empty')
611
612 async def _do_create_files(
613 self,
614 app_data: dict,
615 ) -> None:
616 dest_path = app_data['_instance_dir_path']
617 files = app_data.get('files')
618 if not files:
619 return
620 if not isinstance(files, dict):
621 raise ValueError('"files" must be a dictionary')
622 for filename, data in files.items():
623 path = dest_path / filename
624 if path.is_file():
625 log.info(f"{path} already exists, skipping")
626 with path.open("w") as f:
627 f.write(data.get("content", ""))
628 log.debug(f"{path} created")
629
630 async def start_common(self, app_data: dict) -> None:
631 """Method running common action when starting a manager
632
633 It should be called by managers in "start" method.
634 """
635 await self._do_prepare(app_data)
636 await self._do_create_files(app_data)