diff sat/plugins/plugin_misc_app_manager.py @ 3372:5d926c7b0d99

plugin app manager: first draft: /!\ new optional dependency: pyyaml this plugin manage the life cycle of external applications. Application handlers register to it. Data needed to launch an application as set in YAML files. Local data types are used to get values directly from SàT: - !sat_conf to retrieve a configuration value - !sat_generate_pwd to generate a password - !sat_param for parameters specified a launch Data can be exposed when an instance is launched, this can be used to specify the port (notably the one used for web), or a generated password.
author Goffi <goffi@goffi.org>
date Mon, 28 Sep 2020 21:10:30 +0200
parents
children bfe9ebd253a7
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sat/plugins/plugin_misc_app_manager.py	Mon Sep 28 21:10:30 2020 +0200
@@ -0,0 +1,555 @@
+#!/usr/bin/env python3
+
+# SàT plugin to manage external applications
+# Copyright (C) 2009-2020 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from pathlib import Path
+from typing import Optional, List
+from functools import partial, reduce
+import tempfile
+import secrets
+import string
+import shortuuid
+from twisted.internet import defer
+from twisted.python.procutils import which
+from sat.core.i18n import _
+from sat.core import exceptions
+from sat.core.constants import Const as C
+from sat.core.log import getLogger
+from sat.tools.common import data_format
+from sat.tools.common import async_process
+
+log = getLogger(__name__)
+
+try:
+    import yaml
+except ImportError:
+    raise exceptions.MissingModule(
+        'Missing module PyYAML, please download/install it. You can use '
+        '"pip install pyyaml"'
+    )
+
+try:
+    from yaml import CLoader as Loader, CDumper as Dumper
+except ImportError:
+    log.warning(
+        "Can't use LibYAML binding (is libyaml installed?), pure Python version will be "
+        "used, but it is slower"
+    )
+    from yaml import Loader, Dumper
+
+from yaml.constructor import ConstructorError
+
+
+PLUGIN_INFO = {
+    C.PI_NAME: "Applications Manager",
+    C.PI_IMPORT_NAME: "APP_MANAGER",
+    C.PI_TYPE: C.PLUG_TYPE_MISC,
+    C.PI_MODES: C.PLUG_MODE_BOTH,
+    C.PI_MAIN: "AppManager",
+    C.PI_HANDLER: "no",
+    C.PI_DESCRIPTION: _(
+        """Applications Manager
+
+Manage external applications using packagers, OS virtualization/containers or other
+software management tools.
+"""),
+}
+
+APP_FILE_PREFIX = "sat_app_"
+
+
+class AppManager:
+    load = partial(yaml.load, Loader=Loader)
+    dump = partial(yaml.dump, Dumper=Dumper)
+
+    def __init__(self, host):
+        log.info(_("plugin Applications Manager initialization"))
+        self.host = host
+        self._managers = {}
+        self._apps = {}
+        self._started = {}
+        # instance id to app data map
+        self._instances = {}
+        host.bridge.addMethod(
+            "applicationsList",
+            ".plugin",
+            in_sign="as",
+            out_sign="as",
+            method=self.list_applications,
+        )
+        host.bridge.addMethod(
+            "applicationStart",
+            ".plugin",
+            in_sign="ss",
+            out_sign="",
+            method=self._start,
+            async_=True,
+        )
+        host.bridge.addMethod(
+            "applicationStop",
+            ".plugin",
+            in_sign="sss",
+            out_sign="",
+            method=self._stop,
+            async_=True,
+        )
+        host.bridge.addMethod(
+            "applicationExposedGet",
+            ".plugin",
+            in_sign="sss",
+            out_sign="s",
+            method=self._getExposed,
+            async_=True,
+        )
+        yaml.add_constructor(
+            "!sat_conf", self._sat_conf_constr, Loader=Loader)
+        yaml.add_constructor(
+            "!sat_generate_pwd", self._sat_generate_pwd_constr, Loader=Loader)
+        yaml.add_constructor(
+            "!sat_param", self._sat_param_constr, Loader=Loader)
+
+    def unload(self):
+        log.debug("unloading applications manager")
+        for instances in self._started.values():
+            for instance in instances:
+                data = instance['data']
+                if not data['single_instance']:
+                    log.debug(
+                        f"cleaning temporary directory at {data['_instance_dir_path']}")
+                    data['_instance_dir_obj'].cleanup()
+
+    def _sat_conf_constr(self, loader, node):
+        """Get a value from SàT configuration
+
+        A list is expected with either "name" of a config parameter, a one or more of
+        those parameters:
+            - section
+            - name
+            - default value
+            - filter
+        filter can be:
+            - "first": get the first item of the value
+        """
+        config_data = loader.construct_sequence(node)
+        if len(config_data) == 1:
+            section, name, default, filter_ = "", config_data[0], None, None
+        if len(config_data) == 2:
+            (section, name), default, filter_ = config_data, None, None
+        elif len(config_data) == 3:
+            (section, name, default), filter_ = config_data, None
+        elif len(config_data) == 4:
+            section, name, default, filter_ = config_data
+        else:
+            raise ValueError(
+                f"invalid !sat_conf value ({config_data!r}), a list of 1 to 4 items is "
+                "expected"
+            )
+
+        value = self.host.memory.getConfig(section, name, default)
+        if name == "public_url" and not value or not value.startswith('http'):
+            if not value:
+                log.warning(_(
+                    'No value found for "public_url", using "https://example.org" for '
+                    'now, please set the proper value in sat.conf'))
+            else:
+                log.warning(_(
+                    'invalid value for "public_url" ({value}), it must start with '
+                    '"http", ignoring it and using "https://example.org" instead')
+                        .format(value=value))
+            value = "https://example.org"
+
+        if filter_ is None:
+            pass
+        elif filter_ == 'first':
+            value = value[0]
+        else:
+            raise ValueError(f"unmanaged filter: {filter_}")
+
+        return value
+
+    def _sat_generate_pwd_constr(self, loader, node):
+        alphabet = string.ascii_letters + string.digits
+        return ''.join(secrets.choice(alphabet) for i in range(30))
+
+    def _sat_param_constr(self, loader, node):
+        """Get a parameter specified when starting the application
+
+        The value can be either the name of the parameter to get, or a list as
+        [name, default_value]
+        """
+        try:
+            name, default = loader.construct_sequence(node)
+        except ConstructorError:
+            name, default = loader.construct_scalar(node), None
+        return self._params.get(name, default)
+
+    def register(self, manager):
+        name = manager.name
+        if name in self._managers:
+            raise exceptions.ConflictError(
+                f"There is already a manager with the name {name}")
+        self._managers[manager.name] = manager
+        if hasattr(manager, "discover_path"):
+            self.discover(manager.discover_path, manager)
+
+    def getManager(self, app_data: dict) -> object:
+        """Get manager instance needed for this app
+
+        @raise exceptions.DataError: something is wrong with the type
+        @raise exceptions.NotFound: manager is not registered
+        """
+        try:
+            app_type = app_data["type"]
+        except KeyError:
+            raise exceptions.DataError(
+                "app file doesn't have the mandatory \"type\" key"
+            )
+        if not isinstance(app_type, str):
+            raise exceptions.DataError(
+                f"invalid app data type: {app_type!r}"
+            )
+        app_type = app_type.strip()
+        try:
+            return self._managers[app_type]
+        except KeyError:
+            raise exceptions.NotFound(
+                f"No manager found to manage app of type {app_type!r}")
+
+    def getAppData(
+        self,
+        id_type: Optional[str],
+        identifier: str
+    ) -> dict:
+        """Retrieve instance's app_data from identifier
+
+        @param id_type: type of the identifier, can be:
+            - "name": identifier is a canonical application name
+                the first found instance of this application is returned
+            - "instance": identifier is an instance id
+        @param identifier: identifier according to id_type
+        @return: instance application data
+        @raise exceptions.NotFound: no instance with this id can be found
+        @raise ValueError: id_type is invalid
+        """
+        if not id_type:
+            id_type = 'name'
+        if id_type == 'name':
+            identifier = identifier.lower().strip()
+            try:
+                return next(iter(self._started[identifier]))
+            except (KeyError, StopIteration):
+                raise exceptions.NotFound(
+                    f"No instance of {identifier!r} is currently running"
+                )
+        elif id_type == 'instance':
+            instance_id = identifier
+            try:
+                return self._instances[instance_id]
+            except KeyError:
+                raise exceptions.NotFound(
+                    f"There is no application instance running with id {instance_id!r}"
+                )
+        else:
+            raise ValueError(f"invalid id_type: {id_type!r}")
+
+    def discover(
+            self,
+            dir_path: Path,
+            manager: Optional = None
+    ) -> None:
+        for file_path in dir_path.glob(f"{APP_FILE_PREFIX}*.yaml"):
+            if manager is None:
+                try:
+                    app_data = self.parse(file_path)
+                    manager = self.getManager(app_data)
+                except (exceptions.DataError, exceptions.NotFound) as e:
+                    log.warning(
+                        f"Can't parse {file_path}, skipping: {e}")
+            app_name = file_path.stem[len(APP_FILE_PREFIX):].strip().lower()
+            if not app_name:
+                log.warning(
+                    f"invalid app file name at {file_path}")
+                continue
+            app_dict = self._apps.setdefault(app_name, {})
+            manager_set = app_dict.setdefault(manager, set())
+            manager_set.add(file_path)
+            log.debug(
+                f"{app_name!r} {manager.name} application found"
+            )
+
+    def parse(self, file_path: Path, params: Optional[dict] = None) -> dict:
+        """Parse SàT application file
+
+        @param params: parameters for running this instance
+        @raise exceptions.DataError: something is wrong in the file
+        """
+        if params is None:
+            params = {}
+        with file_path.open() as f:
+            # we set parameters to be used only with this instance
+            # no async method must used between this assignation and `load`
+            self._params = params
+            app_data = self.load(f)
+            self._params = None
+        if "name" not in app_data:
+            # note that we don't use lower() here as we want human readable name and
+            # uppercase may be set on purpose
+            app_data['name'] = file_path.stem[len(APP_FILE_PREFIX):].strip()
+        single_instance = app_data.setdefault("single_instance", True)
+        if not isinstance(single_instance, bool):
+            raise ValueError(
+                f'"single_instance" must be a boolean, but it is {type(single_instance)}'
+            )
+        return app_data
+
+    def list_applications(self, filters: Optional[List[str]]) -> List[str]:
+        """List available application
+
+        @param filters: only show applications matching those filters.
+            using None will list all known applications
+            a filter can be:
+                - available: applications available locally
+                - running: only show launched applications
+        """
+        if not filters:
+            return list(self.apps)
+        found = set()
+        for filter_ in filters:
+            if filter_ == "available":
+                found.update(self._apps)
+            elif filter_ == "running":
+                found.update(self._started)
+            else:
+                raise ValueError(f"Unknown filter: {filter_}")
+        return list(found)
+
+    def _start(self, app_name, extra):
+        extra = data_format.deserialise(extra)
+        return defer.ensureDeferred(self.start(str(app_name), extra))
+
+    async def start(
+        self,
+        app_name: str,
+        extra: Optional[dict] = None,
+    ) -> None:
+        # FIXME: for now we use the first app manager available for the requested app_name
+        # TODO: implement running multiple instance of the same app if some metadata
+        #   to be defined in app_data allows explicitly it.
+        app_name = app_name.lower().strip()
+        try:
+            app_file_path = next(iter(next(iter(self._apps[app_name].values()))))
+        except KeyError:
+            raise exceptions.NotFound(
+                f"No application found with the name {app_name!r}"
+            )
+        started_data = self._started.setdefault(app_name, [])
+        app_data = self.parse(app_file_path, extra)
+        app_data['_file_path'] = app_file_path
+        app_data['_name_canonical'] = app_name
+        single_instance = app_data['single_instance']
+        if single_instance:
+            if started_data:
+                log.info(f"{app_name!r} is already started")
+                return
+            else:
+                cache_path = self.host.memory.getCachePath(
+                    PLUGIN_INFO[C.PI_IMPORT_NAME], app_name
+                )
+                cache_path.mkdir(0o700, parents=True, exist_ok=True)
+                app_data['_instance_dir_path'] = cache_path
+        else:
+            dest_dir_obj = tempfile.TemporaryDirectory(prefix="sat_app_")
+            app_data['_instance_dir_obj'] = dest_dir_obj
+            app_data['_instance_dir_path'] = Path(dest_dir_obj.name)
+        instance_id = app_data['_instance_id'] = shortuuid.uuid()
+        manager = self.getManager(app_data)
+        app_data['_manager'] = manager
+        started_data.append(app_data)
+        self._instances[instance_id] = app_data
+
+        try:
+            start = manager.start
+        except AttributeError:
+            raise exceptions.InternalError(
+                f"{manager.name} doesn't have the mandatory \"start\" method"
+            )
+        else:
+            await start(app_data)
+        log.info(f"{app_name!r} started")
+
+    def _stop(self, identifier, id_type, extra):
+        extra = data_format.deserialise(extra)
+        return defer.ensureDeferred(
+            self.stop(str(identifier), str(id_type) or None, extra))
+
+    async def stop(
+        self,
+        identifier: str,
+        id_type: Optional[str] = None,
+        extra: Optional[dict] = None,
+    ) -> None:
+        if extra is None:
+            extra = {}
+
+        app_data = self.getAppData(id_type, identifier)
+
+        log.info(f"stopping {app_data['name']!r}")
+
+        app_name = app_data['_name_canonical']
+        instance_id = app_data['_instance_id']
+        manager = app_data['_manager']
+
+        try:
+            stop = manager.stop
+        except AttributeError:
+            raise exceptions.InternalError(
+                f"{manager.name} doesn't have the mandatory \"stop\" method"
+            )
+        else:
+            try:
+                await stop(app_data)
+            except Exception as e:
+                log.warning(
+                    f"Instance {instance_id} of application {app_name} can't be stopped "
+                    f"properly: {e}"
+                )
+                return
+
+        try:
+            del self._instances[instance_id]
+        except KeyError:
+            log.error(
+                f"INTERNAL ERROR: {instance_id!r} is not present in self._instances")
+
+        try:
+            self._started[app_name].remove(app_data)
+        except ValueError:
+            log.error(
+                "INTERNAL ERROR: there is no app data in self._started with id "
+                f"{instance_id!r}"
+            )
+
+        log.info(f"{app_name!r} stopped")
+
+    def _getExposed(self, identifier, id_type, extra):
+        extra = data_format.deserialise(extra)
+        d = defer.ensureDeferred(self.getExposed(identifier, id_type, extra))
+        d.addCallback(lambda d: data_format.serialise(d))
+        return d
+
+    def getValueFromPath(self, app_data: dict, path: List[str]) -> any:
+        """Retrieve a value set in the data from it path
+
+        @param path: list of key to use in app_data to retrieve the value
+        @return: found value
+        @raise NotFound: the value can't be found
+        """
+
+    async def getExposed(
+        self,
+        identifier: str,
+        id_type: Optional[str] = None,
+        extra: Optional[dict] = None,
+    ) -> dict:
+        """Get data exposed by the application
+
+        The manager's "computeExpose" method will be called if it exists. It can be used
+        to handle manager specific conventions.
+        """
+        app_data = self.getAppData(id_type, identifier)
+        if app_data.get('_exposed_computed', False):
+            return app_data['expose']
+        if extra is None:
+            extra = {}
+        expose = app_data.setdefault("expose", {})
+        if "passwords" in expose:
+            passwords = expose['passwords']
+            for name, value in list(passwords.items()):
+                if isinstance(value, list):
+                    # if we have a list, is the sequence of keys leading to the value
+                    # to expose. We use "reduce" to retrieve the desired value
+                    try:
+                        passwords[name] = reduce(lambda l, k: l[k], value, app_data)
+                    except Exception as e:
+                        log.warning(
+                            f"Can't retrieve exposed value for password {name!r}: {e}")
+                        del passwords[name]
+
+        url_prefix = expose.get("url_prefix")
+        if isinstance(url_prefix, list):
+            try:
+                expose["url_prefix"] = reduce(lambda l, k: l[k], url_prefix, app_data)
+            except Exception as e:
+                log.warning(
+                    f"Can't retrieve exposed value for url_prefix: {e}")
+                del expose["url_prefix"]
+
+        try:
+            computeExpose = app_data['_manager'].computeExpose
+        except AttributeError:
+            pass
+        else:
+            await computeExpose(app_data)
+
+        app_data['_exposed_computed'] = True
+        return expose
+
+    async def _doPrepare(
+        self,
+        app_data: dict,
+    ) -> None:
+        name = app_data['name']
+        dest_path = app_data['_instance_dir_path']
+        if next(dest_path.iterdir(), None) != None:
+            log.debug(f"There is already a prepared dir at {dest_path}, nothing to do")
+            return
+        try:
+            prepare = app_data['prepare'].copy()
+        except KeyError:
+            prepare = {}
+
+        if not prepare:
+            log.debug("Nothing to prepare for {name!r}")
+            return
+
+        for action, value in list(prepare.items()):
+            log.debug(f"[{name}] [prepare] running {action!r} action")
+            if action == "git":
+                try:
+                    git_path = which('git')[0]
+                except IndexError:
+                    raise exceptions.NotFound(
+                        "Can't find \"git\" executable, {name} can't be started without it"
+                    )
+                await async_process.run(git_path, "clone", value, str(dest_path))
+                log.debug(f"{value!r} git repository cloned at {dest_path}")
+            else:
+                raise NotImplementedError(
+                    f"{action!r} is not managed, can't start {name}"
+                )
+            del prepare[action]
+
+        if prepare:
+            raise exceptions.InternalError('"prepare" should be empty')
+
+    async def startCommon(self, app_data: dict) -> None:
+        """Method running common action when starting a manager
+
+        It should be called by managers in "start" method.
+        """
+        log.info(f"starting {app_data['name']!r}")
+        await self._doPrepare(app_data)