Mercurial > libervia-backend

comparison sat/plugins/plugin_misc_app_manager.py @ 3372:5d926c7b0d99

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
plugin app manager: first draft: /!\ new optional dependency: pyyaml this plugin manage the life cycle of external applications. Application handlers register to it. Data needed to launch an application as set in YAML files. Local data types are used to get values directly from SàT: - !sat_conf to retrieve a configuration value - !sat_generate_pwd to generate a password - !sat_param for parameters specified a launch Data can be exposed when an instance is launched, this can be used to specify the port (notably the one used for web), or a generated password.
author Goffi <goffi@goffi.org>
date Mon, 28 Sep 2020 21:10:30 +0200
parents
children bfe9ebd253a7
comparison
equal deleted inserted replaced
3371:e8d74ac7c479 3372:5d926c7b0d99
1 #!/usr/bin/env python3
2
3 # SàT plugin to manage external applications
4 # Copyright (C) 2009-2020 Jérôme Poisson (goffi@goffi.org)
5
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
10
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
15
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18
19 from pathlib import Path
20 from typing import Optional, List
21 from functools import partial, reduce
22 import tempfile
23 import secrets
24 import string
25 import shortuuid
26 from twisted.internet import defer
27 from twisted.python.procutils import which
28 from sat.core.i18n import _
29 from sat.core import exceptions
30 from sat.core.constants import Const as C
31 from sat.core.log import getLogger
32 from sat.tools.common import data_format
33 from sat.tools.common import async_process
34
35 log = getLogger(__name__)
36
37 try:
38 import yaml
39 except ImportError:
40 raise exceptions.MissingModule(
41 'Missing module PyYAML, please download/install it. You can use '
42 '"pip install pyyaml"'
43 )
44
45 try:
46 from yaml import CLoader as Loader, CDumper as Dumper
47 except ImportError:
48 log.warning(
49 "Can't use LibYAML binding (is libyaml installed?), pure Python version will be "
50 "used, but it is slower"
51 )
52 from yaml import Loader, Dumper
53
54 from yaml.constructor import ConstructorError
55
56
57 PLUGIN_INFO = {
58 C.PI_NAME: "Applications Manager",
59 C.PI_IMPORT_NAME: "APP_MANAGER",
60 C.PI_TYPE: C.PLUG_TYPE_MISC,
61 C.PI_MODES: C.PLUG_MODE_BOTH,
62 C.PI_MAIN: "AppManager",
63 C.PI_HANDLER: "no",
64 C.PI_DESCRIPTION: _(
65 """Applications Manager
66
67 Manage external applications using packagers, OS virtualization/containers or other
68 software management tools.
69 """),
70 }
71
72 APP_FILE_PREFIX = "sat_app_"
73
74
75 class AppManager:
76 load = partial(yaml.load, Loader=Loader)
77 dump = partial(yaml.dump, Dumper=Dumper)
78
79 def __init__(self, host):
80 log.info(_("plugin Applications Manager initialization"))
81 self.host = host
82 self._managers = {}
83 self._apps = {}
84 self._started = {}
85 # instance id to app data map
86 self._instances = {}
87 host.bridge.addMethod(
88 "applicationsList",
89 ".plugin",
90 in_sign="as",
91 out_sign="as",
92 method=self.list_applications,
93 )
94 host.bridge.addMethod(
95 "applicationStart",
96 ".plugin",
97 in_sign="ss",
98 out_sign="",
99 method=self._start,
100 async_=True,
101 )
102 host.bridge.addMethod(
103 "applicationStop",
104 ".plugin",
105 in_sign="sss",
106 out_sign="",
107 method=self._stop,
108 async_=True,
109 )
110 host.bridge.addMethod(
111 "applicationExposedGet",
112 ".plugin",
113 in_sign="sss",
114 out_sign="s",
115 method=self._getExposed,
116 async_=True,
117 )
118 yaml.add_constructor(
119 "!sat_conf", self._sat_conf_constr, Loader=Loader)
120 yaml.add_constructor(
121 "!sat_generate_pwd", self._sat_generate_pwd_constr, Loader=Loader)
122 yaml.add_constructor(
123 "!sat_param", self._sat_param_constr, Loader=Loader)
124
125 def unload(self):
126 log.debug("unloading applications manager")
127 for instances in self._started.values():
128 for instance in instances:
129 data = instance['data']
130 if not data['single_instance']:
131 log.debug(
132 f"cleaning temporary directory at {data['_instance_dir_path']}")
133 data['_instance_dir_obj'].cleanup()
134
135 def _sat_conf_constr(self, loader, node):
136 """Get a value from SàT configuration
137
138 A list is expected with either "name" of a config parameter, a one or more of
139 those parameters:
140 - section
141 - name
142 - default value
143 - filter
144 filter can be:
145 - "first": get the first item of the value
146 """
147 config_data = loader.construct_sequence(node)
148 if len(config_data) == 1:
149 section, name, default, filter_ = "", config_data[0], None, None
150 if len(config_data) == 2:
151 (section, name), default, filter_ = config_data, None, None
152 elif len(config_data) == 3:
153 (section, name, default), filter_ = config_data, None
154 elif len(config_data) == 4:
155 section, name, default, filter_ = config_data
156 else:
157 raise ValueError(
158 f"invalid !sat_conf value ({config_data!r}), a list of 1 to 4 items is "
159 "expected"
160 )
161
162 value = self.host.memory.getConfig(section, name, default)
163 if name == "public_url" and not value or not value.startswith('http'):
164 if not value:
165 log.warning(_(
166 'No value found for "public_url", using "https://example.org" for '
167 'now, please set the proper value in sat.conf'))
168 else:
169 log.warning(_(
170 'invalid value for "public_url" ({value}), it must start with '
171 '"http", ignoring it and using "https://example.org" instead')
172 .format(value=value))
173 value = "https://example.org"
174
175 if filter_ is None:
176 pass
177 elif filter_ == 'first':
178 value = value[0]
179 else:
180 raise ValueError(f"unmanaged filter: {filter_}")
181
182 return value
183
184 def _sat_generate_pwd_constr(self, loader, node):
185 alphabet = string.ascii_letters + string.digits
186 return ''.join(secrets.choice(alphabet) for i in range(30))
187
188 def _sat_param_constr(self, loader, node):
189 """Get a parameter specified when starting the application
190
191 The value can be either the name of the parameter to get, or a list as
192 [name, default_value]
193 """
194 try:
195 name, default = loader.construct_sequence(node)
196 except ConstructorError:
197 name, default = loader.construct_scalar(node), None
198 return self._params.get(name, default)
199
200 def register(self, manager):
201 name = manager.name
202 if name in self._managers:
203 raise exceptions.ConflictError(
204 f"There is already a manager with the name {name}")
205 self._managers[manager.name] = manager
206 if hasattr(manager, "discover_path"):
207 self.discover(manager.discover_path, manager)
208
209 def getManager(self, app_data: dict) -> object:
210 """Get manager instance needed for this app
211
212 @raise exceptions.DataError: something is wrong with the type
213 @raise exceptions.NotFound: manager is not registered
214 """
215 try:
216 app_type = app_data["type"]
217 except KeyError:
218 raise exceptions.DataError(
219 "app file doesn't have the mandatory \"type\" key"
220 )
221 if not isinstance(app_type, str):
222 raise exceptions.DataError(
223 f"invalid app data type: {app_type!r}"
224 )
225 app_type = app_type.strip()
226 try:
227 return self._managers[app_type]
228 except KeyError:
229 raise exceptions.NotFound(
230 f"No manager found to manage app of type {app_type!r}")
231
232 def getAppData(
233 self,
234 id_type: Optional[str],
235 identifier: str
236 ) -> dict:
237 """Retrieve instance's app_data from identifier
238
239 @param id_type: type of the identifier, can be:
240 - "name": identifier is a canonical application name
241 the first found instance of this application is returned
242 - "instance": identifier is an instance id
243 @param identifier: identifier according to id_type
244 @return: instance application data
245 @raise exceptions.NotFound: no instance with this id can be found
246 @raise ValueError: id_type is invalid
247 """
248 if not id_type:
249 id_type = 'name'
250 if id_type == 'name':
251 identifier = identifier.lower().strip()
252 try:
253 return next(iter(self._started[identifier]))
254 except (KeyError, StopIteration):
255 raise exceptions.NotFound(
256 f"No instance of {identifier!r} is currently running"
257 )
258 elif id_type == 'instance':
259 instance_id = identifier
260 try:
261 return self._instances[instance_id]
262 except KeyError:
263 raise exceptions.NotFound(
264 f"There is no application instance running with id {instance_id!r}"
265 )
266 else:
267 raise ValueError(f"invalid id_type: {id_type!r}")
268
269 def discover(
270 self,
271 dir_path: Path,
272 manager: Optional = None
273 ) -> None:
274 for file_path in dir_path.glob(f"{APP_FILE_PREFIX}*.yaml"):
275 if manager is None:
276 try:
277 app_data = self.parse(file_path)
278 manager = self.getManager(app_data)
279 except (exceptions.DataError, exceptions.NotFound) as e:
280 log.warning(
281 f"Can't parse {file_path}, skipping: {e}")
282 app_name = file_path.stem[len(APP_FILE_PREFIX):].strip().lower()
283 if not app_name:
284 log.warning(
285 f"invalid app file name at {file_path}")
286 continue
287 app_dict = self._apps.setdefault(app_name, {})
288 manager_set = app_dict.setdefault(manager, set())
289 manager_set.add(file_path)
290 log.debug(
291 f"{app_name!r} {manager.name} application found"
292 )
293
294 def parse(self, file_path: Path, params: Optional[dict] = None) -> dict:
295 """Parse SàT application file
296
297 @param params: parameters for running this instance
298 @raise exceptions.DataError: something is wrong in the file
299 """
300 if params is None:
301 params = {}
302 with file_path.open() as f:
303 # we set parameters to be used only with this instance
304 # no async method must used between this assignation and `load`
305 self._params = params
306 app_data = self.load(f)
307 self._params = None
308 if "name" not in app_data:
309 # note that we don't use lower() here as we want human readable name and
310 # uppercase may be set on purpose
311 app_data['name'] = file_path.stem[len(APP_FILE_PREFIX):].strip()
312 single_instance = app_data.setdefault("single_instance", True)
313 if not isinstance(single_instance, bool):
314 raise ValueError(
315 f'"single_instance" must be a boolean, but it is {type(single_instance)}'
316 )
317 return app_data
318
319 def list_applications(self, filters: Optional[List[str]]) -> List[str]:
320 """List available application
321
322 @param filters: only show applications matching those filters.
323 using None will list all known applications
324 a filter can be:
325 - available: applications available locally
326 - running: only show launched applications
327 """
328 if not filters:
329 return list(self.apps)
330 found = set()
331 for filter_ in filters:
332 if filter_ == "available":
333 found.update(self._apps)
334 elif filter_ == "running":
335 found.update(self._started)
336 else:
337 raise ValueError(f"Unknown filter: {filter_}")
338 return list(found)
339
340 def _start(self, app_name, extra):
341 extra = data_format.deserialise(extra)
342 return defer.ensureDeferred(self.start(str(app_name), extra))
343
344 async def start(
345 self,
346 app_name: str,
347 extra: Optional[dict] = None,
348 ) -> None:
349 # FIXME: for now we use the first app manager available for the requested app_name
350 # TODO: implement running multiple instance of the same app if some metadata
351 # to be defined in app_data allows explicitly it.
352 app_name = app_name.lower().strip()
353 try:
354 app_file_path = next(iter(next(iter(self._apps[app_name].values()))))
355 except KeyError:
356 raise exceptions.NotFound(
357 f"No application found with the name {app_name!r}"
358 )
359 started_data = self._started.setdefault(app_name, [])
360 app_data = self.parse(app_file_path, extra)
361 app_data['_file_path'] = app_file_path
362 app_data['_name_canonical'] = app_name
363 single_instance = app_data['single_instance']
364 if single_instance:
365 if started_data:
366 log.info(f"{app_name!r} is already started")
367 return
368 else:
369 cache_path = self.host.memory.getCachePath(
370 PLUGIN_INFO[C.PI_IMPORT_NAME], app_name
371 )
372 cache_path.mkdir(0o700, parents=True, exist_ok=True)
373 app_data['_instance_dir_path'] = cache_path
374 else:
375 dest_dir_obj = tempfile.TemporaryDirectory(prefix="sat_app_")
376 app_data['_instance_dir_obj'] = dest_dir_obj
377 app_data['_instance_dir_path'] = Path(dest_dir_obj.name)
378 instance_id = app_data['_instance_id'] = shortuuid.uuid()
379 manager = self.getManager(app_data)
380 app_data['_manager'] = manager
381 started_data.append(app_data)
382 self._instances[instance_id] = app_data
383
384 try:
385 start = manager.start
386 except AttributeError:
387 raise exceptions.InternalError(
388 f"{manager.name} doesn't have the mandatory \"start\" method"
389 )
390 else:
391 await start(app_data)
392 log.info(f"{app_name!r} started")
393
394 def _stop(self, identifier, id_type, extra):
395 extra = data_format.deserialise(extra)
396 return defer.ensureDeferred(
397 self.stop(str(identifier), str(id_type) or None, extra))
398
399 async def stop(
400 self,
401 identifier: str,
402 id_type: Optional[str] = None,
403 extra: Optional[dict] = None,
404 ) -> None:
405 if extra is None:
406 extra = {}
407
408 app_data = self.getAppData(id_type, identifier)
409
410 log.info(f"stopping {app_data['name']!r}")
411
412 app_name = app_data['_name_canonical']
413 instance_id = app_data['_instance_id']
414 manager = app_data['_manager']
415
416 try:
417 stop = manager.stop
418 except AttributeError:
419 raise exceptions.InternalError(
420 f"{manager.name} doesn't have the mandatory \"stop\" method"
421 )
422 else:
423 try:
424 await stop(app_data)
425 except Exception as e:
426 log.warning(
427 f"Instance {instance_id} of application {app_name} can't be stopped "
428 f"properly: {e}"
429 )
430 return
431
432 try:
433 del self._instances[instance_id]
434 except KeyError:
435 log.error(
436 f"INTERNAL ERROR: {instance_id!r} is not present in self._instances")
437
438 try:
439 self._started[app_name].remove(app_data)
440 except ValueError:
441 log.error(
442 "INTERNAL ERROR: there is no app data in self._started with id "
443 f"{instance_id!r}"
444 )
445
446 log.info(f"{app_name!r} stopped")
447
448 def _getExposed(self, identifier, id_type, extra):
449 extra = data_format.deserialise(extra)
450 d = defer.ensureDeferred(self.getExposed(identifier, id_type, extra))
451 d.addCallback(lambda d: data_format.serialise(d))
452 return d
453
454 def getValueFromPath(self, app_data: dict, path: List[str]) -> any:
455 """Retrieve a value set in the data from it path
456
457 @param path: list of key to use in app_data to retrieve the value
458 @return: found value
459 @raise NotFound: the value can't be found
460 """
461
462 async def getExposed(
463 self,
464 identifier: str,
465 id_type: Optional[str] = None,
466 extra: Optional[dict] = None,
467 ) -> dict:
468 """Get data exposed by the application
469
470 The manager's "computeExpose" method will be called if it exists. It can be used
471 to handle manager specific conventions.
472 """
473 app_data = self.getAppData(id_type, identifier)
474 if app_data.get('_exposed_computed', False):
475 return app_data['expose']
476 if extra is None:
477 extra = {}
478 expose = app_data.setdefault("expose", {})
479 if "passwords" in expose:
480 passwords = expose['passwords']
481 for name, value in list(passwords.items()):
482 if isinstance(value, list):
483 # if we have a list, is the sequence of keys leading to the value
484 # to expose. We use "reduce" to retrieve the desired value
485 try:
486 passwords[name] = reduce(lambda l, k: l[k], value, app_data)
487 except Exception as e:
488 log.warning(
489 f"Can't retrieve exposed value for password {name!r}: {e}")
490 del passwords[name]
491
492 url_prefix = expose.get("url_prefix")
493 if isinstance(url_prefix, list):
494 try:
495 expose["url_prefix"] = reduce(lambda l, k: l[k], url_prefix, app_data)
496 except Exception as e:
497 log.warning(
498 f"Can't retrieve exposed value for url_prefix: {e}")
499 del expose["url_prefix"]
500
501 try:
502 computeExpose = app_data['_manager'].computeExpose
503 except AttributeError:
504 pass
505 else:
506 await computeExpose(app_data)
507
508 app_data['_exposed_computed'] = True
509 return expose
510
511 async def _doPrepare(
512 self,
513 app_data: dict,
514 ) -> None:
515 name = app_data['name']
516 dest_path = app_data['_instance_dir_path']
517 if next(dest_path.iterdir(), None) != None:
518 log.debug(f"There is already a prepared dir at {dest_path}, nothing to do")
519 return
520 try:
521 prepare = app_data['prepare'].copy()
522 except KeyError:
523 prepare = {}
524
525 if not prepare:
526 log.debug("Nothing to prepare for {name!r}")
527 return
528
529 for action, value in list(prepare.items()):
530 log.debug(f"[{name}] [prepare] running {action!r} action")
531 if action == "git":
532 try:
533 git_path = which('git')[0]
534 except IndexError:
535 raise exceptions.NotFound(
536 "Can't find \"git\" executable, {name} can't be started without it"
537 )
538 await async_process.run(git_path, "clone", value, str(dest_path))
539 log.debug(f"{value!r} git repository cloned at {dest_path}")
540 else:
541 raise NotImplementedError(
542 f"{action!r} is not managed, can't start {name}"
543 )
544 del prepare[action]
545
546 if prepare:
547 raise exceptions.InternalError('"prepare" should be empty')
548
549 async def startCommon(self, app_data: dict) -> None:
550 """Method running common action when starting a manager
551
552 It should be called by managers in "start" method.
553 """
554 log.info(f"starting {app_data['name']!r}")
555 await self._doPrepare(app_data)