Mercurial > libervia-web
changeset 1146:76d75423ef53
server: tasks manager first draft:
A new task manager will check /tasks directory of website to scripts to execute before launching the site. This allows to generate docs, scripts, or do anything else useful.
Generated files are put in in sat local dir, in cache, and are accessible from the website using the new "build_dir" variable.
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Fri, 25 Jan 2019 08:58:41 +0100 |
| parents | 29eb15062416 |
| children | 02afab1b15c5 |
| files | libervia/server/constants.py libervia/server/pages.py libervia/server/server.py libervia/server/tasks.py |
| diffstat | 4 files changed, 246 insertions(+), 11 deletions(-) [+] |
line wrap: on
line diff
--- a/libervia/server/constants.py Fri Jan 25 08:41:43 2019 +0100 +++ b/libervia/server/constants.py Fri Jan 25 08:58:41 2019 +0100 @@ -33,6 +33,9 @@ MEDIA_DIR = "media/" CARDS_DIR = "games/cards/tarot" PAGES_DIR = u"pages" + TASKS_DIR = u"tasks" + LIBERVIA_CACHE = u"libervia" + BUILD_DIR = u"__b" TPL_RESOURCE = u'_t'
--- a/libervia/server/pages.py Fri Jan 25 08:41:43 2019 +0100 +++ b/libervia/server/pages.py Fri Jan 25 08:58:41 2019 +0100 @@ -392,6 +392,9 @@ ) request._signals_registered.append(signal) + def getBuildPath(self, session_data): + return session_data.cache_dir + self.vhost.site_name + def getPageByName(self, name): return self.vhost_root.getPageByName(name) @@ -1111,6 +1114,7 @@ self.template, media_path=u"/" + C.MEDIA_DIR, cache_path=session_data.cache_dir, + build_path=C.BUILD_DIR + u"/", main_menu=self.main_menu, **template_data) @@ -1308,6 +1312,7 @@ template, media_path="/" + C.MEDIA_DIR, cache_path=session_data.cache_dir, + build_path=C.BUILD_DIR + u"/", main_menu=self.main_menu, **template_data )
--- a/libervia/server/server.py Fri Jan 25 08:41:43 2019 +0100 +++ b/libervia/server/server.py Fri Jan 25 08:58:41 2019 +0100 @@ -17,6 +17,16 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import re +import glob +import os.path +import sys +import tempfile +import shutil +import uuid +import urlparse +import urllib +import time from twisted.application import service from twisted.internet import reactor, defer from twisted.web import server @@ -42,25 +52,16 @@ from sat.core.i18n import _, D_ from sat.core import exceptions from sat.tools import utils +from sat.tools import config from sat.tools.common import regex from sat.tools.common import template from sat.tools.common import uri as common_uri - -import re -import glob -import os.path -import sys -import tempfile -import shutil -import uuid -import urlparse -import urllib -import time from httplib import HTTPS_PORT import libervia from libervia.server import websockets from libervia.server.pages import LiberviaPage from libervia.server.utils import quote, ProgressHandler +from libervia.server.tasks import TasksManager from functools import partial try: @@ -1706,6 +1707,7 @@ self.options = options self.initialised = defer.Deferred() self.waiting_profiles = WaitingRequests() # FIXME: should be removed + self._main_conf = None if self.options["base_url_ext"]: self.base_url_ext = self.options.pop("base_url_ext") @@ -1754,6 +1756,13 @@ roots.insert(0, default) return roots + @property + def main_conf(self): + """SafeConfigParser instance opened on configuration file (sat.conf)""" + if self._main_conf is None: + self._main_conf = config.parseMainConf() + return self._main_conf + def _namespacesGetCb(self, ns_map): self.ns_map = ns_map @@ -1794,6 +1803,7 @@ if default_dict: conf[u''] = default_dict + @defer.inlineCallbacks def backendReady(self, __): self.media_dir = self.bridge.getConfig("", "media_dir") self.local_dir = self.bridge.getConfig("", "local_dir") @@ -1813,6 +1823,8 @@ self.sat_root = default_root = LiberviaRootResource( host=self, host_name=u'', site_name=u'', site_path=default_site_path, path=self.html_dir) + tasks_manager = TasksManager(self, self.sat_root) + yield tasks_manager.runTasks() LiberviaPage.importPages(self, self.sat_root) # FIXME: handle _setMenu in a more generic way, taking care of external sites self.sat_root._setMenu(self.options["menu_json"]) @@ -1844,6 +1856,9 @@ site_name=site_name, site_path=site_path, path=root_path) + tasks_manager = TasksManager(self, res) + yield tasks_manager.runTasks() + res.putChild(C.BUILD_DIR, static.File(self.getBuildPath(site_name))) self.vhost_root.addHost(host_name.encode('utf-8'), res) LiberviaPage.importPages(self, res) # FIXME: default pages are accessible if not overriden by external website @@ -2361,6 +2376,20 @@ for root in self.roots: root.putChild(path, wrapped_res) + def getBuildPath(self, site_name): + """Generate build path for a given site name + + @param site_name(unicode): name of the site + @return (unicode): path to the build directory + """ + build_path_elts = [ + config.getConfig(self.main_conf, "", "local_dir"), + C.CACHE_DIR, + C.LIBERVIA_CACHE, + regex.pathEscape(site_name)] + build_path = u"/".join(build_path_elts) + return os.path.abspath(os.path.expanduser(build_path)) + def getExtBaseURLData(self, request): """Retrieve external base URL Data
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libervia/server/tasks.py Fri Jan 25 08:58:41 2019 +0100 @@ -0,0 +1,198 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Libervia: a Salut à Toi frontend +# Copyright (C) 2011-2019 Jérôme Poisson <goffi@goffi.org> + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import os +import os.path +from twisted.internet import defer +from twisted.python.procutils import which +from sat.core import exceptions +from sat.core.i18n import _ +from libervia.server.constants import Const as C +from collections import OrderedDict +from sat.core.log import getLogger +from sat.tools import config +from sat.tools.common import async_process + +log = getLogger(__name__) + + +class TasksManager(object): + """Handle tasks of a Libervia site""" + FILE_EXTS = {u'py'} + + def __init__(self, host, site_resource): + """ + @param site_resource(LiberviaRootResource): root resource of the site to manage + """ + self.host = host + self.resource = site_resource + self.tasks_dir = os.path.join(self.resource.site_path, C.TASKS_DIR) + self.tasks = OrderedDict() + self.parseTasks() + self._build_path = None + self._current_task = None + + @property + def site_path(self): + return self.resource.site_path + + @property + def config_section(self): + return self.resource.site_name.lower().strip() + + @property + def build_path(self): + """path where generated files will be build for this site""" + if self._build_path is None: + self._build_path = self.host.getBuildPath(self.site_name) + return self._build_path + + def getConfig(self, key, default=None, value_type=None): + """Retrieve configuration associated to this site + + Section is automatically set to site name + @param key(unicode): key to use + @param default: value to use if not found (see [config.getConfig]) + @param value_type(unicode, None): filter to use on value + Note that filters are already automatically used when the key finish + by a well known suffix ("_path", "_list", "_dict", or "_json") + None to use no filter, else can be: + - "path": a path is expected, will be normalized and expanded + + """ + value = config.getConfig(self.host.main_conf, self.config_section, key, + default=default) + if value_type is not None: + if value_type == u'path': + v_filter = lambda v: os.path.abspath(os.path.expanduser(v)) + else: + raise ValueError(u"unknown value type {value_type}".format( + value_type = value_type)) + if isinstance(value, list): + value = [v_filter(v) for v in value] + elif isinstance(value, dict): + value = {k:v_filter(v) for k,v in value.items()} + elif value is not None: + value = v_filter(v) + return value + + @property + def site_name(self): + return self.resource.site_name + + @property + def task_data(self): + return self.tasks[self._current_task][u'data'] + + def validateData(self, data): + """Check values in data""" + + for var, default, allowed in ((u"ON_ERROR", u"stop", (u"continue", u"stop")), + (u"LOG_OUTPUT", True, bool)): + value = data.setdefault(var, default) + if isinstance(allowed, type): + if not isinstance(value, allowed): + raise ValueError( + _(u"Unexpected value for {var}, {allowed} is expected.") + .format(var = var, allowed = allowed)) + else: + if not value in allowed: + raise ValueError(_(u"Unexpected value for {var}: {value}").format( + var = var, value = value)) + + for var, default, allowed in [[u"ON_ERROR", u"stop", (u"continue", u"stop")]]: + value = data.setdefault(var, default) + if not value in allowed: + raise ValueError(_(u"Unexpected value for {var}: {value}").format( + var = var, value = value)) + + def parseTasks(self): + if not os.path.isdir(self.tasks_dir): + log.debug(_(u"{name} has no task to launch.").format( + name = self.resource.site_name or u"default site")) + return + filenames = os.listdir(self.tasks_dir) + filenames.sort() + for filename in filenames: + filepath = os.path.join(self.tasks_dir, filename) + if not filename.startswith(u'task_') or not os.path.isfile(filepath): + continue + task_name, ext = os.path.splitext(filename) + task_name = task_name[5:].lower().strip() + if not task_name: + continue + if ext[1:] not in self.FILE_EXTS: + continue + if task_name in self.tasks: + raise exceptions.ConflictError( + u"A task with the name [{name}] already exists".format( + name=task_name)) + task_data = {u"__name__": "{site_name}.task.{name}".format( + site_name=self.site_name, name=task_name)} + self.tasks[task_name] = { + u'path': filepath, + u'data': task_data, + } + execfile(filepath, task_data) + self.validateData(task_data) + + @defer.inlineCallbacks + def runTasks(self): + """Run all the tasks found""" + old_path = os.getcwd() + for task_name, task_value in self.tasks.iteritems(): + self._current_task = task_name + log.info(_(u'== running task "{task_name}" for {site_name} =='.format( + task_name=task_name, site_name=self.site_name))) + data = task_value[u'data'] + os.chdir(self.site_path) + try: + yield data['start'](self) + except Exception as e: + on_error = data[u'ON_ERROR'] + if on_error == u'stop': + raise e + elif on_error == u'continue': + log.warning(_(u'Task "{task_name}" failed for {site_name}: {reason}') + .format(task_name = task_name, site_name = self.site_name, reason = e)) + else: + raise exceptions.InternalError(u"we should never reach this point") + self._current_task = None + os.chdir(old_path) + + def findCommand(self, name, *args): + """Find full path of a shell command + + @param name(unicode): name of the command to find + @param *args(unicode): extra names the command may have + @return (unicode): full path of the command + @raise exceptions.NotFound: can't find this command + """ + names = (name,) + args + for n in names: + try: + cmd_path = which(n)[0].encode('utf-8') + except IndexError: + pass + return cmd_path + raise exceptions.NotFound(_( + u"Can't find {name} command, did you install it?").format(name=name)) + + def runCommand(self, command, *args, **kwargs): + kwargs['verbose'] = self.task_data[u"LOG_OUTPUT"] + return async_process.CommandProtocol.run(command, *args, **kwargs)