|
3372
|
1 #!/usr/bin/env python3 |
|
|
2 |
|
|
3 # SàT plugin to manage external applications |
|
|
4 # Copyright (C) 2009-2020 Jérôme Poisson (goffi@goffi.org) |
|
|
5 |
|
|
6 # This program is free software: you can redistribute it and/or modify |
|
|
7 # it under the terms of the GNU Affero General Public License as published by |
|
|
8 # the Free Software Foundation, either version 3 of the License, or |
|
|
9 # (at your option) any later version. |
|
|
10 |
|
|
11 # This program is distributed in the hope that it will be useful, |
|
|
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
|
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
|
14 # GNU Affero General Public License for more details. |
|
|
15 |
|
|
16 # You should have received a copy of the GNU Affero General Public License |
|
|
17 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
|
18 |
|
|
19 from pathlib import Path |
|
|
20 from typing import Optional, List |
|
|
21 from functools import partial, reduce |
|
|
22 import tempfile |
|
|
23 import secrets |
|
|
24 import string |
|
|
25 import shortuuid |
|
|
26 from twisted.internet import defer |
|
|
27 from twisted.python.procutils import which |
|
|
28 from sat.core.i18n import _ |
|
|
29 from sat.core import exceptions |
|
|
30 from sat.core.constants import Const as C |
|
|
31 from sat.core.log import getLogger |
|
|
32 from sat.tools.common import data_format |
|
|
33 from sat.tools.common import async_process |
|
|
34 |
|
|
35 log = getLogger(__name__) |
|
|
36 |
|
|
37 try: |
|
|
38 import yaml |
|
|
39 except ImportError: |
|
|
40 raise exceptions.MissingModule( |
|
|
41 'Missing module PyYAML, please download/install it. You can use ' |
|
|
42 '"pip install pyyaml"' |
|
|
43 ) |
|
|
44 |
|
|
45 try: |
|
|
46 from yaml import CLoader as Loader, CDumper as Dumper |
|
|
47 except ImportError: |
|
|
48 log.warning( |
|
|
49 "Can't use LibYAML binding (is libyaml installed?), pure Python version will be " |
|
|
50 "used, but it is slower" |
|
|
51 ) |
|
|
52 from yaml import Loader, Dumper |
|
|
53 |
|
|
54 from yaml.constructor import ConstructorError |
|
|
55 |
|
|
56 |
|
|
57 PLUGIN_INFO = { |
|
|
58 C.PI_NAME: "Applications Manager", |
|
|
59 C.PI_IMPORT_NAME: "APP_MANAGER", |
|
|
60 C.PI_TYPE: C.PLUG_TYPE_MISC, |
|
|
61 C.PI_MODES: C.PLUG_MODE_BOTH, |
|
|
62 C.PI_MAIN: "AppManager", |
|
|
63 C.PI_HANDLER: "no", |
|
|
64 C.PI_DESCRIPTION: _( |
|
|
65 """Applications Manager |
|
|
66 |
|
|
67 Manage external applications using packagers, OS virtualization/containers or other |
|
|
68 software management tools. |
|
|
69 """), |
|
|
70 } |
|
|
71 |
|
|
72 APP_FILE_PREFIX = "sat_app_" |
|
|
73 |
|
|
74 |
|
|
75 class AppManager: |
|
|
76 load = partial(yaml.load, Loader=Loader) |
|
|
77 dump = partial(yaml.dump, Dumper=Dumper) |
|
|
78 |
|
|
79 def __init__(self, host): |
|
|
80 log.info(_("plugin Applications Manager initialization")) |
|
|
81 self.host = host |
|
|
82 self._managers = {} |
|
|
83 self._apps = {} |
|
|
84 self._started = {} |
|
|
85 # instance id to app data map |
|
|
86 self._instances = {} |
|
|
87 host.bridge.addMethod( |
|
|
88 "applicationsList", |
|
|
89 ".plugin", |
|
|
90 in_sign="as", |
|
|
91 out_sign="as", |
|
|
92 method=self.list_applications, |
|
|
93 ) |
|
|
94 host.bridge.addMethod( |
|
|
95 "applicationStart", |
|
|
96 ".plugin", |
|
|
97 in_sign="ss", |
|
|
98 out_sign="", |
|
|
99 method=self._start, |
|
|
100 async_=True, |
|
|
101 ) |
|
|
102 host.bridge.addMethod( |
|
|
103 "applicationStop", |
|
|
104 ".plugin", |
|
|
105 in_sign="sss", |
|
|
106 out_sign="", |
|
|
107 method=self._stop, |
|
|
108 async_=True, |
|
|
109 ) |
|
|
110 host.bridge.addMethod( |
|
|
111 "applicationExposedGet", |
|
|
112 ".plugin", |
|
|
113 in_sign="sss", |
|
|
114 out_sign="s", |
|
|
115 method=self._getExposed, |
|
|
116 async_=True, |
|
|
117 ) |
|
|
118 yaml.add_constructor( |
|
|
119 "!sat_conf", self._sat_conf_constr, Loader=Loader) |
|
|
120 yaml.add_constructor( |
|
|
121 "!sat_generate_pwd", self._sat_generate_pwd_constr, Loader=Loader) |
|
|
122 yaml.add_constructor( |
|
|
123 "!sat_param", self._sat_param_constr, Loader=Loader) |
|
|
124 |
|
|
125 def unload(self): |
|
|
126 log.debug("unloading applications manager") |
|
|
127 for instances in self._started.values(): |
|
|
128 for instance in instances: |
|
|
129 data = instance['data'] |
|
|
130 if not data['single_instance']: |
|
|
131 log.debug( |
|
|
132 f"cleaning temporary directory at {data['_instance_dir_path']}") |
|
|
133 data['_instance_dir_obj'].cleanup() |
|
|
134 |
|
|
135 def _sat_conf_constr(self, loader, node): |
|
|
136 """Get a value from SàT configuration |
|
|
137 |
|
|
138 A list is expected with either "name" of a config parameter, a one or more of |
|
|
139 those parameters: |
|
|
140 - section |
|
|
141 - name |
|
|
142 - default value |
|
|
143 - filter |
|
|
144 filter can be: |
|
|
145 - "first": get the first item of the value |
|
|
146 """ |
|
|
147 config_data = loader.construct_sequence(node) |
|
|
148 if len(config_data) == 1: |
|
|
149 section, name, default, filter_ = "", config_data[0], None, None |
|
|
150 if len(config_data) == 2: |
|
|
151 (section, name), default, filter_ = config_data, None, None |
|
|
152 elif len(config_data) == 3: |
|
|
153 (section, name, default), filter_ = config_data, None |
|
|
154 elif len(config_data) == 4: |
|
|
155 section, name, default, filter_ = config_data |
|
|
156 else: |
|
|
157 raise ValueError( |
|
|
158 f"invalid !sat_conf value ({config_data!r}), a list of 1 to 4 items is " |
|
|
159 "expected" |
|
|
160 ) |
|
|
161 |
|
|
162 value = self.host.memory.getConfig(section, name, default) |
|
|
163 if name == "public_url" and not value or not value.startswith('http'): |
|
|
164 if not value: |
|
|
165 log.warning(_( |
|
|
166 'No value found for "public_url", using "https://example.org" for ' |
|
|
167 'now, please set the proper value in sat.conf')) |
|
|
168 else: |
|
|
169 log.warning(_( |
|
|
170 'invalid value for "public_url" ({value}), it must start with ' |
|
|
171 '"http", ignoring it and using "https://example.org" instead') |
|
|
172 .format(value=value)) |
|
|
173 value = "https://example.org" |
|
|
174 |
|
|
175 if filter_ is None: |
|
|
176 pass |
|
|
177 elif filter_ == 'first': |
|
|
178 value = value[0] |
|
|
179 else: |
|
|
180 raise ValueError(f"unmanaged filter: {filter_}") |
|
|
181 |
|
|
182 return value |
|
|
183 |
|
|
184 def _sat_generate_pwd_constr(self, loader, node): |
|
|
185 alphabet = string.ascii_letters + string.digits |
|
|
186 return ''.join(secrets.choice(alphabet) for i in range(30)) |
|
|
187 |
|
|
188 def _sat_param_constr(self, loader, node): |
|
|
189 """Get a parameter specified when starting the application |
|
|
190 |
|
|
191 The value can be either the name of the parameter to get, or a list as |
|
|
192 [name, default_value] |
|
|
193 """ |
|
|
194 try: |
|
|
195 name, default = loader.construct_sequence(node) |
|
|
196 except ConstructorError: |
|
|
197 name, default = loader.construct_scalar(node), None |
|
|
198 return self._params.get(name, default) |
|
|
199 |
|
|
200 def register(self, manager): |
|
|
201 name = manager.name |
|
|
202 if name in self._managers: |
|
|
203 raise exceptions.ConflictError( |
|
|
204 f"There is already a manager with the name {name}") |
|
|
205 self._managers[manager.name] = manager |
|
|
206 if hasattr(manager, "discover_path"): |
|
|
207 self.discover(manager.discover_path, manager) |
|
|
208 |
|
|
209 def getManager(self, app_data: dict) -> object: |
|
|
210 """Get manager instance needed for this app |
|
|
211 |
|
|
212 @raise exceptions.DataError: something is wrong with the type |
|
|
213 @raise exceptions.NotFound: manager is not registered |
|
|
214 """ |
|
|
215 try: |
|
|
216 app_type = app_data["type"] |
|
|
217 except KeyError: |
|
|
218 raise exceptions.DataError( |
|
|
219 "app file doesn't have the mandatory \"type\" key" |
|
|
220 ) |
|
|
221 if not isinstance(app_type, str): |
|
|
222 raise exceptions.DataError( |
|
|
223 f"invalid app data type: {app_type!r}" |
|
|
224 ) |
|
|
225 app_type = app_type.strip() |
|
|
226 try: |
|
|
227 return self._managers[app_type] |
|
|
228 except KeyError: |
|
|
229 raise exceptions.NotFound( |
|
|
230 f"No manager found to manage app of type {app_type!r}") |
|
|
231 |
|
|
232 def getAppData( |
|
|
233 self, |
|
|
234 id_type: Optional[str], |
|
|
235 identifier: str |
|
|
236 ) -> dict: |
|
|
237 """Retrieve instance's app_data from identifier |
|
|
238 |
|
|
239 @param id_type: type of the identifier, can be: |
|
|
240 - "name": identifier is a canonical application name |
|
|
241 the first found instance of this application is returned |
|
|
242 - "instance": identifier is an instance id |
|
|
243 @param identifier: identifier according to id_type |
|
|
244 @return: instance application data |
|
|
245 @raise exceptions.NotFound: no instance with this id can be found |
|
|
246 @raise ValueError: id_type is invalid |
|
|
247 """ |
|
|
248 if not id_type: |
|
|
249 id_type = 'name' |
|
|
250 if id_type == 'name': |
|
|
251 identifier = identifier.lower().strip() |
|
|
252 try: |
|
|
253 return next(iter(self._started[identifier])) |
|
|
254 except (KeyError, StopIteration): |
|
|
255 raise exceptions.NotFound( |
|
|
256 f"No instance of {identifier!r} is currently running" |
|
|
257 ) |
|
|
258 elif id_type == 'instance': |
|
|
259 instance_id = identifier |
|
|
260 try: |
|
|
261 return self._instances[instance_id] |
|
|
262 except KeyError: |
|
|
263 raise exceptions.NotFound( |
|
|
264 f"There is no application instance running with id {instance_id!r}" |
|
|
265 ) |
|
|
266 else: |
|
|
267 raise ValueError(f"invalid id_type: {id_type!r}") |
|
|
268 |
|
|
269 def discover( |
|
|
270 self, |
|
|
271 dir_path: Path, |
|
|
272 manager: Optional = None |
|
|
273 ) -> None: |
|
|
274 for file_path in dir_path.glob(f"{APP_FILE_PREFIX}*.yaml"): |
|
|
275 if manager is None: |
|
|
276 try: |
|
|
277 app_data = self.parse(file_path) |
|
|
278 manager = self.getManager(app_data) |
|
|
279 except (exceptions.DataError, exceptions.NotFound) as e: |
|
|
280 log.warning( |
|
|
281 f"Can't parse {file_path}, skipping: {e}") |
|
|
282 app_name = file_path.stem[len(APP_FILE_PREFIX):].strip().lower() |
|
|
283 if not app_name: |
|
|
284 log.warning( |
|
|
285 f"invalid app file name at {file_path}") |
|
|
286 continue |
|
|
287 app_dict = self._apps.setdefault(app_name, {}) |
|
|
288 manager_set = app_dict.setdefault(manager, set()) |
|
|
289 manager_set.add(file_path) |
|
|
290 log.debug( |
|
|
291 f"{app_name!r} {manager.name} application found" |
|
|
292 ) |
|
|
293 |
|
|
294 def parse(self, file_path: Path, params: Optional[dict] = None) -> dict: |
|
|
295 """Parse SàT application file |
|
|
296 |
|
|
297 @param params: parameters for running this instance |
|
|
298 @raise exceptions.DataError: something is wrong in the file |
|
|
299 """ |
|
|
300 if params is None: |
|
|
301 params = {} |
|
|
302 with file_path.open() as f: |
|
|
303 # we set parameters to be used only with this instance |
|
|
304 # no async method must used between this assignation and `load` |
|
|
305 self._params = params |
|
|
306 app_data = self.load(f) |
|
|
307 self._params = None |
|
|
308 if "name" not in app_data: |
|
|
309 # note that we don't use lower() here as we want human readable name and |
|
|
310 # uppercase may be set on purpose |
|
|
311 app_data['name'] = file_path.stem[len(APP_FILE_PREFIX):].strip() |
|
|
312 single_instance = app_data.setdefault("single_instance", True) |
|
|
313 if not isinstance(single_instance, bool): |
|
|
314 raise ValueError( |
|
|
315 f'"single_instance" must be a boolean, but it is {type(single_instance)}' |
|
|
316 ) |
|
|
317 return app_data |
|
|
318 |
|
|
319 def list_applications(self, filters: Optional[List[str]]) -> List[str]: |
|
|
320 """List available application |
|
|
321 |
|
|
322 @param filters: only show applications matching those filters. |
|
|
323 using None will list all known applications |
|
|
324 a filter can be: |
|
|
325 - available: applications available locally |
|
|
326 - running: only show launched applications |
|
|
327 """ |
|
|
328 if not filters: |
|
|
329 return list(self.apps) |
|
|
330 found = set() |
|
|
331 for filter_ in filters: |
|
|
332 if filter_ == "available": |
|
|
333 found.update(self._apps) |
|
|
334 elif filter_ == "running": |
|
|
335 found.update(self._started) |
|
|
336 else: |
|
|
337 raise ValueError(f"Unknown filter: {filter_}") |
|
|
338 return list(found) |
|
|
339 |
|
|
340 def _start(self, app_name, extra): |
|
|
341 extra = data_format.deserialise(extra) |
|
|
342 return defer.ensureDeferred(self.start(str(app_name), extra)) |
|
|
343 |
|
|
344 async def start( |
|
|
345 self, |
|
|
346 app_name: str, |
|
|
347 extra: Optional[dict] = None, |
|
|
348 ) -> None: |
|
|
349 # FIXME: for now we use the first app manager available for the requested app_name |
|
|
350 # TODO: implement running multiple instance of the same app if some metadata |
|
|
351 # to be defined in app_data allows explicitly it. |
|
|
352 app_name = app_name.lower().strip() |
|
|
353 try: |
|
|
354 app_file_path = next(iter(next(iter(self._apps[app_name].values())))) |
|
|
355 except KeyError: |
|
|
356 raise exceptions.NotFound( |
|
|
357 f"No application found with the name {app_name!r}" |
|
|
358 ) |
|
|
359 started_data = self._started.setdefault(app_name, []) |
|
|
360 app_data = self.parse(app_file_path, extra) |
|
|
361 app_data['_file_path'] = app_file_path |
|
|
362 app_data['_name_canonical'] = app_name |
|
|
363 single_instance = app_data['single_instance'] |
|
|
364 if single_instance: |
|
|
365 if started_data: |
|
|
366 log.info(f"{app_name!r} is already started") |
|
|
367 return |
|
|
368 else: |
|
|
369 cache_path = self.host.memory.getCachePath( |
|
|
370 PLUGIN_INFO[C.PI_IMPORT_NAME], app_name |
|
|
371 ) |
|
|
372 cache_path.mkdir(0o700, parents=True, exist_ok=True) |
|
|
373 app_data['_instance_dir_path'] = cache_path |
|
|
374 else: |
|
|
375 dest_dir_obj = tempfile.TemporaryDirectory(prefix="sat_app_") |
|
|
376 app_data['_instance_dir_obj'] = dest_dir_obj |
|
|
377 app_data['_instance_dir_path'] = Path(dest_dir_obj.name) |
|
|
378 instance_id = app_data['_instance_id'] = shortuuid.uuid() |
|
|
379 manager = self.getManager(app_data) |
|
|
380 app_data['_manager'] = manager |
|
|
381 started_data.append(app_data) |
|
|
382 self._instances[instance_id] = app_data |
|
|
383 |
|
|
384 try: |
|
|
385 start = manager.start |
|
|
386 except AttributeError: |
|
|
387 raise exceptions.InternalError( |
|
|
388 f"{manager.name} doesn't have the mandatory \"start\" method" |
|
|
389 ) |
|
|
390 else: |
|
|
391 await start(app_data) |
|
|
392 log.info(f"{app_name!r} started") |
|
|
393 |
|
|
394 def _stop(self, identifier, id_type, extra): |
|
|
395 extra = data_format.deserialise(extra) |
|
|
396 return defer.ensureDeferred( |
|
|
397 self.stop(str(identifier), str(id_type) or None, extra)) |
|
|
398 |
|
|
399 async def stop( |
|
|
400 self, |
|
|
401 identifier: str, |
|
|
402 id_type: Optional[str] = None, |
|
|
403 extra: Optional[dict] = None, |
|
|
404 ) -> None: |
|
|
405 if extra is None: |
|
|
406 extra = {} |
|
|
407 |
|
|
408 app_data = self.getAppData(id_type, identifier) |
|
|
409 |
|
|
410 log.info(f"stopping {app_data['name']!r}") |
|
|
411 |
|
|
412 app_name = app_data['_name_canonical'] |
|
|
413 instance_id = app_data['_instance_id'] |
|
|
414 manager = app_data['_manager'] |
|
|
415 |
|
|
416 try: |
|
|
417 stop = manager.stop |
|
|
418 except AttributeError: |
|
|
419 raise exceptions.InternalError( |
|
|
420 f"{manager.name} doesn't have the mandatory \"stop\" method" |
|
|
421 ) |
|
|
422 else: |
|
|
423 try: |
|
|
424 await stop(app_data) |
|
|
425 except Exception as e: |
|
|
426 log.warning( |
|
|
427 f"Instance {instance_id} of application {app_name} can't be stopped " |
|
|
428 f"properly: {e}" |
|
|
429 ) |
|
|
430 return |
|
|
431 |
|
|
432 try: |
|
|
433 del self._instances[instance_id] |
|
|
434 except KeyError: |
|
|
435 log.error( |
|
|
436 f"INTERNAL ERROR: {instance_id!r} is not present in self._instances") |
|
|
437 |
|
|
438 try: |
|
|
439 self._started[app_name].remove(app_data) |
|
|
440 except ValueError: |
|
|
441 log.error( |
|
|
442 "INTERNAL ERROR: there is no app data in self._started with id " |
|
|
443 f"{instance_id!r}" |
|
|
444 ) |
|
|
445 |
|
|
446 log.info(f"{app_name!r} stopped") |
|
|
447 |
|
|
448 def _getExposed(self, identifier, id_type, extra): |
|
|
449 extra = data_format.deserialise(extra) |
|
|
450 d = defer.ensureDeferred(self.getExposed(identifier, id_type, extra)) |
|
|
451 d.addCallback(lambda d: data_format.serialise(d)) |
|
|
452 return d |
|
|
453 |
|
|
454 def getValueFromPath(self, app_data: dict, path: List[str]) -> any: |
|
|
455 """Retrieve a value set in the data from it path |
|
|
456 |
|
|
457 @param path: list of key to use in app_data to retrieve the value |
|
|
458 @return: found value |
|
|
459 @raise NotFound: the value can't be found |
|
|
460 """ |
|
|
461 |
|
|
462 async def getExposed( |
|
|
463 self, |
|
|
464 identifier: str, |
|
|
465 id_type: Optional[str] = None, |
|
|
466 extra: Optional[dict] = None, |
|
|
467 ) -> dict: |
|
|
468 """Get data exposed by the application |
|
|
469 |
|
|
470 The manager's "computeExpose" method will be called if it exists. It can be used |
|
|
471 to handle manager specific conventions. |
|
|
472 """ |
|
|
473 app_data = self.getAppData(id_type, identifier) |
|
|
474 if app_data.get('_exposed_computed', False): |
|
|
475 return app_data['expose'] |
|
|
476 if extra is None: |
|
|
477 extra = {} |
|
|
478 expose = app_data.setdefault("expose", {}) |
|
|
479 if "passwords" in expose: |
|
|
480 passwords = expose['passwords'] |
|
|
481 for name, value in list(passwords.items()): |
|
|
482 if isinstance(value, list): |
|
|
483 # if we have a list, is the sequence of keys leading to the value |
|
|
484 # to expose. We use "reduce" to retrieve the desired value |
|
|
485 try: |
|
|
486 passwords[name] = reduce(lambda l, k: l[k], value, app_data) |
|
|
487 except Exception as e: |
|
|
488 log.warning( |
|
|
489 f"Can't retrieve exposed value for password {name!r}: {e}") |
|
|
490 del passwords[name] |
|
|
491 |
|
|
492 url_prefix = expose.get("url_prefix") |
|
|
493 if isinstance(url_prefix, list): |
|
|
494 try: |
|
|
495 expose["url_prefix"] = reduce(lambda l, k: l[k], url_prefix, app_data) |
|
|
496 except Exception as e: |
|
|
497 log.warning( |
|
|
498 f"Can't retrieve exposed value for url_prefix: {e}") |
|
|
499 del expose["url_prefix"] |
|
|
500 |
|
|
501 try: |
|
|
502 computeExpose = app_data['_manager'].computeExpose |
|
|
503 except AttributeError: |
|
|
504 pass |
|
|
505 else: |
|
|
506 await computeExpose(app_data) |
|
|
507 |
|
|
508 app_data['_exposed_computed'] = True |
|
|
509 return expose |
|
|
510 |
|
|
511 async def _doPrepare( |
|
|
512 self, |
|
|
513 app_data: dict, |
|
|
514 ) -> None: |
|
|
515 name = app_data['name'] |
|
|
516 dest_path = app_data['_instance_dir_path'] |
|
|
517 if next(dest_path.iterdir(), None) != None: |
|
|
518 log.debug(f"There is already a prepared dir at {dest_path}, nothing to do") |
|
|
519 return |
|
|
520 try: |
|
|
521 prepare = app_data['prepare'].copy() |
|
|
522 except KeyError: |
|
|
523 prepare = {} |
|
|
524 |
|
|
525 if not prepare: |
|
|
526 log.debug("Nothing to prepare for {name!r}") |
|
|
527 return |
|
|
528 |
|
|
529 for action, value in list(prepare.items()): |
|
|
530 log.debug(f"[{name}] [prepare] running {action!r} action") |
|
|
531 if action == "git": |
|
|
532 try: |
|
|
533 git_path = which('git')[0] |
|
|
534 except IndexError: |
|
|
535 raise exceptions.NotFound( |
|
|
536 "Can't find \"git\" executable, {name} can't be started without it" |
|
|
537 ) |
|
|
538 await async_process.run(git_path, "clone", value, str(dest_path)) |
|
|
539 log.debug(f"{value!r} git repository cloned at {dest_path}") |
|
|
540 else: |
|
|
541 raise NotImplementedError( |
|
|
542 f"{action!r} is not managed, can't start {name}" |
|
|
543 ) |
|
|
544 del prepare[action] |
|
|
545 |
|
|
546 if prepare: |
|
|
547 raise exceptions.InternalError('"prepare" should be empty') |
|
|
548 |
|
|
549 async def startCommon(self, app_data: dict) -> None: |
|
|
550 """Method running common action when starting a manager |
|
|
551 |
|
|
552 It should be called by managers in "start" method. |
|
|
553 """ |
|
|
554 log.info(f"starting {app_data['name']!r}") |
|
|
555 await self._doPrepare(app_data) |
