Mercurial > libervia-backend
annotate sat/plugins/plugin_misc_app_manager.py @ 3479:be6d91572633
date update
author | Goffi <goffi@goffi.org> |
---|---|
date | Fri, 19 Mar 2021 15:45:14 +0100 |
parents | bfe9ebd253a7 |
children | 2c9e95796371 |
rev | line source |
---|---|
3372 | 1 #!/usr/bin/env python3 |
2 | |
3 # SàT plugin to manage external applications | |
3479 | 4 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) |
3372 | 5 |
6 # This program is free software: you can redistribute it and/or modify | |
7 # it under the terms of the GNU Affero General Public License as published by | |
8 # the Free Software Foundation, either version 3 of the License, or | |
9 # (at your option) any later version. | |
10 | |
11 # This program is distributed in the hope that it will be useful, | |
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 # GNU Affero General Public License for more details. | |
15 | |
16 # You should have received a copy of the GNU Affero General Public License | |
17 # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | |
19 from pathlib import Path | |
20 from typing import Optional, List | |
21 from functools import partial, reduce | |
22 import tempfile | |
23 import secrets | |
24 import string | |
25 import shortuuid | |
26 from twisted.internet import defer | |
27 from twisted.python.procutils import which | |
28 from sat.core.i18n import _ | |
29 from sat.core import exceptions | |
30 from sat.core.constants import Const as C | |
31 from sat.core.log import getLogger | |
32 from sat.tools.common import data_format | |
33 from sat.tools.common import async_process | |
34 | |
35 log = getLogger(__name__) | |
36 | |
37 try: | |
38 import yaml | |
39 except ImportError: | |
40 raise exceptions.MissingModule( | |
41 'Missing module PyYAML, please download/install it. You can use ' | |
42 '"pip install pyyaml"' | |
43 ) | |
44 | |
45 try: | |
46 from yaml import CLoader as Loader, CDumper as Dumper | |
47 except ImportError: | |
48 log.warning( | |
49 "Can't use LibYAML binding (is libyaml installed?), pure Python version will be " | |
50 "used, but it is slower" | |
51 ) | |
52 from yaml import Loader, Dumper | |
53 | |
54 from yaml.constructor import ConstructorError | |
55 | |
56 | |
57 PLUGIN_INFO = { | |
58 C.PI_NAME: "Applications Manager", | |
59 C.PI_IMPORT_NAME: "APP_MANAGER", | |
60 C.PI_TYPE: C.PLUG_TYPE_MISC, | |
61 C.PI_MODES: C.PLUG_MODE_BOTH, | |
62 C.PI_MAIN: "AppManager", | |
63 C.PI_HANDLER: "no", | |
64 C.PI_DESCRIPTION: _( | |
65 """Applications Manager | |
66 | |
67 Manage external applications using packagers, OS virtualization/containers or other | |
68 software management tools. | |
69 """), | |
70 } | |
71 | |
72 APP_FILE_PREFIX = "sat_app_" | |
73 | |
74 | |
75 class AppManager: | |
76 load = partial(yaml.load, Loader=Loader) | |
77 dump = partial(yaml.dump, Dumper=Dumper) | |
78 | |
79 def __init__(self, host): | |
80 log.info(_("plugin Applications Manager initialization")) | |
81 self.host = host | |
82 self._managers = {} | |
83 self._apps = {} | |
84 self._started = {} | |
85 # instance id to app data map | |
86 self._instances = {} | |
87 host.bridge.addMethod( | |
88 "applicationsList", | |
89 ".plugin", | |
90 in_sign="as", | |
91 out_sign="as", | |
92 method=self.list_applications, | |
93 ) | |
94 host.bridge.addMethod( | |
95 "applicationStart", | |
96 ".plugin", | |
97 in_sign="ss", | |
98 out_sign="", | |
99 method=self._start, | |
100 async_=True, | |
101 ) | |
102 host.bridge.addMethod( | |
103 "applicationStop", | |
104 ".plugin", | |
105 in_sign="sss", | |
106 out_sign="", | |
107 method=self._stop, | |
108 async_=True, | |
109 ) | |
110 host.bridge.addMethod( | |
111 "applicationExposedGet", | |
112 ".plugin", | |
113 in_sign="sss", | |
114 out_sign="s", | |
115 method=self._getExposed, | |
116 async_=True, | |
117 ) | |
118 yaml.add_constructor( | |
119 "!sat_conf", self._sat_conf_constr, Loader=Loader) | |
120 yaml.add_constructor( | |
121 "!sat_generate_pwd", self._sat_generate_pwd_constr, Loader=Loader) | |
122 yaml.add_constructor( | |
123 "!sat_param", self._sat_param_constr, Loader=Loader) | |
124 | |
125 def unload(self): | |
126 log.debug("unloading applications manager") | |
127 for instances in self._started.values(): | |
128 for instance in instances: | |
129 data = instance['data'] | |
130 if not data['single_instance']: | |
131 log.debug( | |
132 f"cleaning temporary directory at {data['_instance_dir_path']}") | |
133 data['_instance_dir_obj'].cleanup() | |
134 | |
135 def _sat_conf_constr(self, loader, node): | |
136 """Get a value from SàT configuration | |
137 | |
138 A list is expected with either "name" of a config parameter, a one or more of | |
139 those parameters: | |
140 - section | |
141 - name | |
142 - default value | |
143 - filter | |
144 filter can be: | |
145 - "first": get the first item of the value | |
146 """ | |
147 config_data = loader.construct_sequence(node) | |
148 if len(config_data) == 1: | |
149 section, name, default, filter_ = "", config_data[0], None, None | |
150 if len(config_data) == 2: | |
151 (section, name), default, filter_ = config_data, None, None | |
152 elif len(config_data) == 3: | |
153 (section, name, default), filter_ = config_data, None | |
154 elif len(config_data) == 4: | |
155 section, name, default, filter_ = config_data | |
156 else: | |
157 raise ValueError( | |
158 f"invalid !sat_conf value ({config_data!r}), a list of 1 to 4 items is " | |
159 "expected" | |
160 ) | |
161 | |
162 value = self.host.memory.getConfig(section, name, default) | |
3417
bfe9ebd253a7
plugin app manager: fixed `public_url` value check
Goffi <goffi@goffi.org>
parents:
3372
diff
changeset
|
163 if name == "public_url" and (not value or not value.startswith('http')): |
3372 | 164 if not value: |
165 log.warning(_( | |
166 'No value found for "public_url", using "https://example.org" for ' | |
167 'now, please set the proper value in sat.conf')) | |
168 else: | |
169 log.warning(_( | |
170 'invalid value for "public_url" ({value}), it must start with ' | |
171 '"http", ignoring it and using "https://example.org" instead') | |
172 .format(value=value)) | |
173 value = "https://example.org" | |
174 | |
175 if filter_ is None: | |
176 pass | |
177 elif filter_ == 'first': | |
178 value = value[0] | |
179 else: | |
180 raise ValueError(f"unmanaged filter: {filter_}") | |
181 | |
182 return value | |
183 | |
184 def _sat_generate_pwd_constr(self, loader, node): | |
185 alphabet = string.ascii_letters + string.digits | |
186 return ''.join(secrets.choice(alphabet) for i in range(30)) | |
187 | |
188 def _sat_param_constr(self, loader, node): | |
189 """Get a parameter specified when starting the application | |
190 | |
191 The value can be either the name of the parameter to get, or a list as | |
192 [name, default_value] | |
193 """ | |
194 try: | |
195 name, default = loader.construct_sequence(node) | |
196 except ConstructorError: | |
197 name, default = loader.construct_scalar(node), None | |
198 return self._params.get(name, default) | |
199 | |
200 def register(self, manager): | |
201 name = manager.name | |
202 if name in self._managers: | |
203 raise exceptions.ConflictError( | |
204 f"There is already a manager with the name {name}") | |
205 self._managers[manager.name] = manager | |
206 if hasattr(manager, "discover_path"): | |
207 self.discover(manager.discover_path, manager) | |
208 | |
209 def getManager(self, app_data: dict) -> object: | |
210 """Get manager instance needed for this app | |
211 | |
212 @raise exceptions.DataError: something is wrong with the type | |
213 @raise exceptions.NotFound: manager is not registered | |
214 """ | |
215 try: | |
216 app_type = app_data["type"] | |
217 except KeyError: | |
218 raise exceptions.DataError( | |
219 "app file doesn't have the mandatory \"type\" key" | |
220 ) | |
221 if not isinstance(app_type, str): | |
222 raise exceptions.DataError( | |
223 f"invalid app data type: {app_type!r}" | |
224 ) | |
225 app_type = app_type.strip() | |
226 try: | |
227 return self._managers[app_type] | |
228 except KeyError: | |
229 raise exceptions.NotFound( | |
230 f"No manager found to manage app of type {app_type!r}") | |
231 | |
232 def getAppData( | |
233 self, | |
234 id_type: Optional[str], | |
235 identifier: str | |
236 ) -> dict: | |
237 """Retrieve instance's app_data from identifier | |
238 | |
239 @param id_type: type of the identifier, can be: | |
240 - "name": identifier is a canonical application name | |
241 the first found instance of this application is returned | |
242 - "instance": identifier is an instance id | |
243 @param identifier: identifier according to id_type | |
244 @return: instance application data | |
245 @raise exceptions.NotFound: no instance with this id can be found | |
246 @raise ValueError: id_type is invalid | |
247 """ | |
248 if not id_type: | |
249 id_type = 'name' | |
250 if id_type == 'name': | |
251 identifier = identifier.lower().strip() | |
252 try: | |
253 return next(iter(self._started[identifier])) | |
254 except (KeyError, StopIteration): | |
255 raise exceptions.NotFound( | |
256 f"No instance of {identifier!r} is currently running" | |
257 ) | |
258 elif id_type == 'instance': | |
259 instance_id = identifier | |
260 try: | |
261 return self._instances[instance_id] | |
262 except KeyError: | |
263 raise exceptions.NotFound( | |
264 f"There is no application instance running with id {instance_id!r}" | |
265 ) | |
266 else: | |
267 raise ValueError(f"invalid id_type: {id_type!r}") | |
268 | |
269 def discover( | |
270 self, | |
271 dir_path: Path, | |
272 manager: Optional = None | |
273 ) -> None: | |
274 for file_path in dir_path.glob(f"{APP_FILE_PREFIX}*.yaml"): | |
275 if manager is None: | |
276 try: | |
277 app_data = self.parse(file_path) | |
278 manager = self.getManager(app_data) | |
279 except (exceptions.DataError, exceptions.NotFound) as e: | |
280 log.warning( | |
281 f"Can't parse {file_path}, skipping: {e}") | |
282 app_name = file_path.stem[len(APP_FILE_PREFIX):].strip().lower() | |
283 if not app_name: | |
284 log.warning( | |
285 f"invalid app file name at {file_path}") | |
286 continue | |
287 app_dict = self._apps.setdefault(app_name, {}) | |
288 manager_set = app_dict.setdefault(manager, set()) | |
289 manager_set.add(file_path) | |
290 log.debug( | |
291 f"{app_name!r} {manager.name} application found" | |
292 ) | |
293 | |
294 def parse(self, file_path: Path, params: Optional[dict] = None) -> dict: | |
295 """Parse SàT application file | |
296 | |
297 @param params: parameters for running this instance | |
298 @raise exceptions.DataError: something is wrong in the file | |
299 """ | |
300 if params is None: | |
301 params = {} | |
302 with file_path.open() as f: | |
303 # we set parameters to be used only with this instance | |
304 # no async method must used between this assignation and `load` | |
305 self._params = params | |
306 app_data = self.load(f) | |
307 self._params = None | |
308 if "name" not in app_data: | |
309 # note that we don't use lower() here as we want human readable name and | |
310 # uppercase may be set on purpose | |
311 app_data['name'] = file_path.stem[len(APP_FILE_PREFIX):].strip() | |
312 single_instance = app_data.setdefault("single_instance", True) | |
313 if not isinstance(single_instance, bool): | |
314 raise ValueError( | |
315 f'"single_instance" must be a boolean, but it is {type(single_instance)}' | |
316 ) | |
317 return app_data | |
318 | |
319 def list_applications(self, filters: Optional[List[str]]) -> List[str]: | |
320 """List available application | |
321 | |
322 @param filters: only show applications matching those filters. | |
323 using None will list all known applications | |
324 a filter can be: | |
325 - available: applications available locally | |
326 - running: only show launched applications | |
327 """ | |
328 if not filters: | |
329 return list(self.apps) | |
330 found = set() | |
331 for filter_ in filters: | |
332 if filter_ == "available": | |
333 found.update(self._apps) | |
334 elif filter_ == "running": | |
335 found.update(self._started) | |
336 else: | |
337 raise ValueError(f"Unknown filter: {filter_}") | |
338 return list(found) | |
339 | |
340 def _start(self, app_name, extra): | |
341 extra = data_format.deserialise(extra) | |
342 return defer.ensureDeferred(self.start(str(app_name), extra)) | |
343 | |
344 async def start( | |
345 self, | |
346 app_name: str, | |
347 extra: Optional[dict] = None, | |
348 ) -> None: | |
349 # FIXME: for now we use the first app manager available for the requested app_name | |
350 # TODO: implement running multiple instance of the same app if some metadata | |
351 # to be defined in app_data allows explicitly it. | |
352 app_name = app_name.lower().strip() | |
353 try: | |
354 app_file_path = next(iter(next(iter(self._apps[app_name].values())))) | |
355 except KeyError: | |
356 raise exceptions.NotFound( | |
357 f"No application found with the name {app_name!r}" | |
358 ) | |
359 started_data = self._started.setdefault(app_name, []) | |
360 app_data = self.parse(app_file_path, extra) | |
361 app_data['_file_path'] = app_file_path | |
362 app_data['_name_canonical'] = app_name | |
363 single_instance = app_data['single_instance'] | |
364 if single_instance: | |
365 if started_data: | |
366 log.info(f"{app_name!r} is already started") | |
367 return | |
368 else: | |
369 cache_path = self.host.memory.getCachePath( | |
370 PLUGIN_INFO[C.PI_IMPORT_NAME], app_name | |
371 ) | |
372 cache_path.mkdir(0o700, parents=True, exist_ok=True) | |
373 app_data['_instance_dir_path'] = cache_path | |
374 else: | |
375 dest_dir_obj = tempfile.TemporaryDirectory(prefix="sat_app_") | |
376 app_data['_instance_dir_obj'] = dest_dir_obj | |
377 app_data['_instance_dir_path'] = Path(dest_dir_obj.name) | |
378 instance_id = app_data['_instance_id'] = shortuuid.uuid() | |
379 manager = self.getManager(app_data) | |
380 app_data['_manager'] = manager | |
381 started_data.append(app_data) | |
382 self._instances[instance_id] = app_data | |
383 | |
384 try: | |
385 start = manager.start | |
386 except AttributeError: | |
387 raise exceptions.InternalError( | |
388 f"{manager.name} doesn't have the mandatory \"start\" method" | |
389 ) | |
390 else: | |
391 await start(app_data) | |
392 log.info(f"{app_name!r} started") | |
393 | |
394 def _stop(self, identifier, id_type, extra): | |
395 extra = data_format.deserialise(extra) | |
396 return defer.ensureDeferred( | |
397 self.stop(str(identifier), str(id_type) or None, extra)) | |
398 | |
399 async def stop( | |
400 self, | |
401 identifier: str, | |
402 id_type: Optional[str] = None, | |
403 extra: Optional[dict] = None, | |
404 ) -> None: | |
405 if extra is None: | |
406 extra = {} | |
407 | |
408 app_data = self.getAppData(id_type, identifier) | |
409 | |
410 log.info(f"stopping {app_data['name']!r}") | |
411 | |
412 app_name = app_data['_name_canonical'] | |
413 instance_id = app_data['_instance_id'] | |
414 manager = app_data['_manager'] | |
415 | |
416 try: | |
417 stop = manager.stop | |
418 except AttributeError: | |
419 raise exceptions.InternalError( | |
420 f"{manager.name} doesn't have the mandatory \"stop\" method" | |
421 ) | |
422 else: | |
423 try: | |
424 await stop(app_data) | |
425 except Exception as e: | |
426 log.warning( | |
427 f"Instance {instance_id} of application {app_name} can't be stopped " | |
428 f"properly: {e}" | |
429 ) | |
430 return | |
431 | |
432 try: | |
433 del self._instances[instance_id] | |
434 except KeyError: | |
435 log.error( | |
436 f"INTERNAL ERROR: {instance_id!r} is not present in self._instances") | |
437 | |
438 try: | |
439 self._started[app_name].remove(app_data) | |
440 except ValueError: | |
441 log.error( | |
442 "INTERNAL ERROR: there is no app data in self._started with id " | |
443 f"{instance_id!r}" | |
444 ) | |
445 | |
446 log.info(f"{app_name!r} stopped") | |
447 | |
448 def _getExposed(self, identifier, id_type, extra): | |
449 extra = data_format.deserialise(extra) | |
450 d = defer.ensureDeferred(self.getExposed(identifier, id_type, extra)) | |
451 d.addCallback(lambda d: data_format.serialise(d)) | |
452 return d | |
453 | |
454 def getValueFromPath(self, app_data: dict, path: List[str]) -> any: | |
455 """Retrieve a value set in the data from it path | |
456 | |
457 @param path: list of key to use in app_data to retrieve the value | |
458 @return: found value | |
459 @raise NotFound: the value can't be found | |
460 """ | |
461 | |
462 async def getExposed( | |
463 self, | |
464 identifier: str, | |
465 id_type: Optional[str] = None, | |
466 extra: Optional[dict] = None, | |
467 ) -> dict: | |
468 """Get data exposed by the application | |
469 | |
470 The manager's "computeExpose" method will be called if it exists. It can be used | |
471 to handle manager specific conventions. | |
472 """ | |
473 app_data = self.getAppData(id_type, identifier) | |
474 if app_data.get('_exposed_computed', False): | |
475 return app_data['expose'] | |
476 if extra is None: | |
477 extra = {} | |
478 expose = app_data.setdefault("expose", {}) | |
479 if "passwords" in expose: | |
480 passwords = expose['passwords'] | |
481 for name, value in list(passwords.items()): | |
482 if isinstance(value, list): | |
483 # if we have a list, is the sequence of keys leading to the value | |
484 # to expose. We use "reduce" to retrieve the desired value | |
485 try: | |
486 passwords[name] = reduce(lambda l, k: l[k], value, app_data) | |
487 except Exception as e: | |
488 log.warning( | |
489 f"Can't retrieve exposed value for password {name!r}: {e}") | |
490 del passwords[name] | |
491 | |
492 url_prefix = expose.get("url_prefix") | |
493 if isinstance(url_prefix, list): | |
494 try: | |
495 expose["url_prefix"] = reduce(lambda l, k: l[k], url_prefix, app_data) | |
496 except Exception as e: | |
497 log.warning( | |
498 f"Can't retrieve exposed value for url_prefix: {e}") | |
499 del expose["url_prefix"] | |
500 | |
501 try: | |
502 computeExpose = app_data['_manager'].computeExpose | |
503 except AttributeError: | |
504 pass | |
505 else: | |
506 await computeExpose(app_data) | |
507 | |
508 app_data['_exposed_computed'] = True | |
509 return expose | |
510 | |
511 async def _doPrepare( | |
512 self, | |
513 app_data: dict, | |
514 ) -> None: | |
515 name = app_data['name'] | |
516 dest_path = app_data['_instance_dir_path'] | |
517 if next(dest_path.iterdir(), None) != None: | |
518 log.debug(f"There is already a prepared dir at {dest_path}, nothing to do") | |
519 return | |
520 try: | |
521 prepare = app_data['prepare'].copy() | |
522 except KeyError: | |
523 prepare = {} | |
524 | |
525 if not prepare: | |
526 log.debug("Nothing to prepare for {name!r}") | |
527 return | |
528 | |
529 for action, value in list(prepare.items()): | |
530 log.debug(f"[{name}] [prepare] running {action!r} action") | |
531 if action == "git": | |
532 try: | |
533 git_path = which('git')[0] | |
534 except IndexError: | |
535 raise exceptions.NotFound( | |
536 "Can't find \"git\" executable, {name} can't be started without it" | |
537 ) | |
538 await async_process.run(git_path, "clone", value, str(dest_path)) | |
539 log.debug(f"{value!r} git repository cloned at {dest_path}") | |
540 else: | |
541 raise NotImplementedError( | |
542 f"{action!r} is not managed, can't start {name}" | |
543 ) | |
544 del prepare[action] | |
545 | |
546 if prepare: | |
547 raise exceptions.InternalError('"prepare" should be empty') | |
548 | |
549 async def startCommon(self, app_data: dict) -> None: | |
550 """Method running common action when starting a manager | |
551 | |
552 It should be called by managers in "start" method. | |
553 """ | |
554 log.info(f"starting {app_data['name']!r}") | |
555 await self._doPrepare(app_data) |