--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/libervia/backend/tools/common/template.py	Fri Jun 02 11:49:51 2023 +0200
@@ -0,0 +1,1064 @@
+#!/usr/bin/env python3
+
+# SAT: an XMPP client
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Template generation"""
+
+import os.path
+import time
+import re
+import json
+from datetime import datetime
+from pathlib import Path
+from collections import namedtuple
+from typing import Optional, List, Tuple, Union
+from xml.sax.saxutils import quoteattr
+from babel import support
+from babel import Locale
+from babel.core import UnknownLocaleError
+import pygments
+from pygments import lexers
+from pygments import formatters
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.i18n import _
+from libervia.backend.core import exceptions
+from libervia.backend.tools import config
+from libervia.backend.tools.common import date_utils
+from libervia.backend.core.log import getLogger
+
+log = getLogger(__name__)
+
+try:
+    import sat_templates
+except ImportError:
+    raise exceptions.MissingModule(
+        "sat_templates module is not available, please install it or check your path to "
+        "use template engine"
+    )
+else:
+    sat_templates  # to avoid pyflakes warning
+
+try:
+    import jinja2
+except:
+    raise exceptions.MissingModule(
+        "Missing module jinja2, please install it from http://jinja.pocoo.org or with "
+        "pip install jinja2"
+    )
+
+from lxml import etree
+from jinja2 import Markup as safe
+from jinja2 import is_undefined
+from jinja2 import utils
+from jinja2 import TemplateNotFound
+from jinja2 import contextfilter
+from jinja2.loaders import split_template_path
+
+HTML_EXT = ("html", "xhtml")
+RE_ATTR_ESCAPE = re.compile(r"[^a-z_-]")
+SITE_RESERVED_NAMES = ("sat",)
+TPL_RESERVED_CHARS = r"()/."
+RE_TPL_RESERVED_CHARS = re.compile("[" + TPL_RESERVED_CHARS + "]")
+BROWSER_DIR = "_browser"
+BROWSER_META_FILE = "browser_meta.json"
+
+TemplateData = namedtuple("TemplateData", ['site', 'theme', 'path'])
+
+
+class TemplateLoader(jinja2.BaseLoader):
+    """A template loader which handle site, theme and absolute paths"""
+    # TODO: list_templates should be implemented
+
+    def __init__(self, sites_paths, sites_themes, trusted=False):
+        """
+        @param trusted(bool): if True, absolue template paths will be allowed
+            be careful when using this option and sure that you can trust the template,
+            as this allow the template to open any file on the system that the
+            launching user can access.
+        """
+        if not sites_paths or not "" in sites_paths:
+            raise exceptions.InternalError("Invalid sites_paths")
+        super(jinja2.BaseLoader, self).__init__()
+        self.sites_paths = sites_paths
+        self.sites_themes = sites_themes
+        self.trusted = trusted
+
+    @staticmethod
+    def parse_template(template):
+        """Parse template path and return site, theme and path
+
+        @param template_path(unicode): path to template with parenthesis syntax
+            The site and/or theme can be specified in parenthesis just before the path
+            e.g.: (some_theme)path/to/template.html
+                  (/some_theme)path/to/template.html (equivalent to previous one)
+                  (other_site/other_theme)path/to/template.html
+                  (other_site/)path/to/template.html (defaut theme for other_site)
+                  /absolute/path/to/template.html (in trusted environment only)
+        @return (TemplateData):
+            site, theme and template_path.
+            if site is empty, SàT Templates are used
+            site and theme can be both None if absolute path is used
+            Relative path is the path from theme root dir e.g. blog/articles.html
+        """
+        if template.startswith("("):
+            # site and/or theme are specified
+            try:
+                theme_end = template.index(")")
+            except IndexError:
+                raise ValueError("incorrect site/theme in template")
+            theme_data = template[1:theme_end]
+            theme_splitted = theme_data.split('/')
+            if len(theme_splitted) == 1:
+                site, theme = "", theme_splitted[0]
+            elif len(theme_splitted) == 2:
+                site, theme = theme_splitted
+            else:
+                raise ValueError("incorrect site/theme in template")
+            template_path = template[theme_end+1:]
+            if not template_path or template_path.startswith("/"):
+                raise ValueError("incorrect template path")
+        elif template.startswith("/"):
+            # this is an absolute path, so we have no site and no theme
+            site = None
+            theme = None
+            template_path = template
+        else:
+            # a default template
+            site = ""
+            theme = C.TEMPLATE_THEME_DEFAULT
+            template_path = template
+
+        if site is not None:
+            site = site.strip()
+            if not site:
+                site = ""
+            elif site in SITE_RESERVED_NAMES:
+                raise ValueError(_("{site} can't be used as site name, "
+                                   "it's reserved.").format(site=site))
+
+        if theme is not None:
+            theme = theme.strip()
+            if not theme:
+                theme = C.TEMPLATE_THEME_DEFAULT
+            if RE_TPL_RESERVED_CHARS.search(theme):
+                raise ValueError(_("{theme} contain forbidden char. Following chars "
+                                   "are forbidden: {reserved}").format(
+                                   theme=theme, reserved=TPL_RESERVED_CHARS))
+
+        return TemplateData(site, theme, template_path)
+
+    @staticmethod
+    def get_sites_and_themes(
+            site: str,
+            theme: str,
+            settings: Optional[dict] = None,
+        ) -> List[Tuple[str, str]]:
+        """Get sites and themes to check for template/file
+
+        Will add default theme and default site in search list when suitable. Settings'
+        `fallback` can be used to modify behaviour: themes in this list will then be used
+        instead of default (it can also be empty list or None, in which case no fallback
+        is used).
+
+        @param site: site requested
+        @param theme: theme requested
+        @return: site and theme couples to check
+        """
+        if settings is None:
+            settings = {}
+        sites_and_themes = [[site, theme]]
+        fallback = settings.get("fallback", [C.TEMPLATE_THEME_DEFAULT])
+        for fb_theme in fallback:
+            if theme != fb_theme:
+                sites_and_themes.append([site, fb_theme])
+        if site:
+            for fb_theme in fallback:
+                sites_and_themes.append(["", fb_theme])
+        return sites_and_themes
+
+    def _get_template_f(self, site, theme, path_elts):
+        """Look for template and return opened file if found
+
+        @param site(unicode): names of site to check
+            (default site will also checked)
+        @param theme(unicode): theme to check (default theme will also be checked)
+        @param path_elts(iterable[str]): elements of template path
+        @return (tuple[(File, None), (str, None)]): a tuple with:
+            - opened template, or None if not found
+            - absolute file path, or None if not found
+        """
+        if site is None:
+            raise exceptions.InternalError(
+                "_get_template_f must not be used with absolute path")
+        settings = self.sites_themes[site][theme]['settings']
+        for site_to_check, theme_to_check in self.get_sites_and_themes(
+                site, theme, settings):
+            try:
+                base_path = self.sites_paths[site_to_check]
+            except KeyError:
+                log.warning(_("Unregistered site requested: {site_to_check}").format(
+                    site_to_check=site_to_check))
+            filepath = os.path.join(
+                base_path,
+                C.TEMPLATE_TPL_DIR,
+                theme_to_check,
+                *path_elts
+            )
+            f = utils.open_if_exists(filepath, 'r')
+            if f is not None:
+                return f, filepath
+        return None, None
+
+    def get_source(self, environment, template):
+        """Retrieve source handling site and themes
+
+        If the path is absolute it is used directly if in trusted environment
+        else and exception is raised.
+        if the path is just relative, "default" theme is used.
+        @raise PermissionError: absolute path used in untrusted environment
+        """
+        site, theme, template_path = self.parse_template(template)
+
+        if site is None:
+            # we have an abolute template
+            if theme is not None:
+                raise exceptions.InternalError("We can't have a theme with absolute "
+                                               "template.")
+            if not self.trusted:
+                log.error(_("Absolute template used while unsecure is disabled, hack "
+                            "attempt? Template: {template}").format(template=template))
+                raise exceptions.PermissionError("absolute template is not allowed")
+            filepath = template_path
+            f = utils.open_if_exists(filepath, 'r')
+        else:
+            # relative path, we have to deal with site and theme
+            assert theme and template_path
+            path_elts = split_template_path(template_path)
+            # if we have non default site, we check it first, else we only check default
+            f, filepath = self._get_template_f(site, theme, path_elts)
+
+        if f is None:
+            if (site is not None and path_elts[0] == "error"
+                and os.path.splitext(template_path)[1][1:] in HTML_EXT):
+                # if an HTML error is requested but doesn't exist, we try again
+                # with base error.
+                f, filepath = self._get_template_f(
+                    site, theme, ("error", "base.html"))
+                if f is None:
+                    raise exceptions.InternalError("error/base.html should exist")
+            else:
+                raise TemplateNotFound(template)
+
+        try:
+            contents = f.read()
+        finally:
+            f.close()
+
+        mtime = os.path.getmtime(filepath)
+
+        def uptodate():
+            try:
+                return os.path.getmtime(filepath) == mtime
+            except OSError:
+                return False
+
+        return contents, filepath, uptodate
+
+
+class Indexer(object):
+    """Index global to a page"""
+
+    def __init__(self):
+        self._indexes = {}
+
+    def next(self, value):
+        if value not in self._indexes:
+            self._indexes[value] = 0
+            return 0
+        self._indexes[value] += 1
+        return self._indexes[value]
+
+    def current(self, value):
+        return self._indexes.get(value)
+
+
+class ScriptsHandler(object):
+    def __init__(self, renderer, template_data):
+        self.renderer = renderer
+        self.template_data = template_data
+        self.scripts = []  #  we don't use a set because order may be important
+
+    def include(self, library_name, attribute="defer"):
+        """Mark that a script need to be imported.
+
+        Must be used before base.html is extended, as <script> are generated there.
+        If called several time with the same library, it will be imported once.
+        @param library_name(unicode): name of the library to import
+        @param loading:
+        """
+        if attribute not in ("defer", "async", ""):
+            raise exceptions.DataError(
+                _('Invalid attribute, please use one of "defer", "async" or ""')
+            )
+        if not library_name.endswith(".js"):
+            library_name = library_name + ".js"
+        if (library_name, attribute) not in self.scripts:
+            self.scripts.append((library_name, attribute))
+        return ""
+
+    def generate_scripts(self):
+        """Generate the <script> elements
+
+        @return (unicode): <scripts> HTML tags
+        """
+        scripts = []
+        tpl = "<script src={src} {attribute}></script>"
+        for library, attribute in self.scripts:
+            library_path = self.renderer.get_static_path(self.template_data, library)
+            if library_path is None:
+                log.warning(_("Can't find {libary} javascript library").format(
+                    library=library))
+                continue
+            path = self.renderer.get_front_url(library_path)
+            scripts.append(tpl.format(src=quoteattr(path), attribute=attribute))
+        return safe("\n".join(scripts))
+
+
+class Environment(jinja2.Environment):
+
+    def get_template(self, name, parent=None, globals=None):
+        if name[0] not in ('/', '('):
+            # if name is not an absolute path or a full template name (this happen on
+            # extend or import during rendering), we convert it to a full template name.
+            # This is needed to handle cache correctly when a base template is overriden.
+            # Without that, we could not distinguish something like base/base.html if
+            # it's launched from some_site/some_theme or from [default]/default
+            name = "({site}/{theme}){template}".format(
+                site=self._template_data.site,
+                theme=self._template_data.theme,
+                template=name)
+
+        return super(Environment, self).get_template(name, parent, globals)
+
+
+class Renderer(object):
+
+    def __init__(self, host, front_url_filter=None, trusted=False, private=False):
+        """
+        @param front_url_filter(callable): filter to retrieve real url of a directory/file
+            The callable will get a two arguments:
+                - a dict with a "template_data" key containing TemplateData instance of
+                  current template. Only site and theme should be necessary.
+                - the relative URL of the file to retrieve, relative from theme root
+            None to use default filter which return real path on file
+            Need to be specified for web rendering, to reflect URL seen by end user
+        @param trusted(bool): if True, allow to access absolute path
+            Only set to True if environment is safe (e.g. command line tool)
+        @param private(bool): if True, also load sites from sites_path_private_dict
+        """
+        self.host = host
+        self.trusted = trusted
+        self.sites_paths = {
+            "": os.path.dirname(sat_templates.__file__),
+        }
+        self.sites_themes = {
+        }
+        conf = config.parse_main_conf()
+        public_sites = config.config_get(conf, None, "sites_path_public_dict", {})
+        sites_data = [public_sites]
+        if private:
+            private_sites = config.config_get(conf, None, "sites_path_private_dict", {})
+            sites_data.append(private_sites)
+        for sites in sites_data:
+            normalised = {}
+            for name, path in sites.items():
+                if RE_TPL_RESERVED_CHARS.search(name):
+                    log.warning(_("Can't add \"{name}\" site, it contains forbidden "
+                                  "characters. Forbidden characters are {forbidden}.")
+                                .format(name=name, forbidden=TPL_RESERVED_CHARS))
+                    continue
+                path = os.path.expanduser(os.path.normpath(path))
+                if not path or not path.startswith("/"):
+                    log.warning(_("Can't add \"{name}\" site, it should map to an "
+                                  "absolute path").format(name=name))
+                    continue
+                normalised[name] = path
+            self.sites_paths.update(normalised)
+
+        for site, site_path in self.sites_paths.items():
+            tpl_path = Path(site_path) / C.TEMPLATE_TPL_DIR
+            for p in tpl_path.iterdir():
+                if not p.is_dir():
+                    continue
+                log.debug(f"theme found for {site or 'default site'}: {p.name}")
+                theme_data = self.sites_themes.setdefault(site, {})[p.name] = {
+                    'path': p,
+                    'settings': {}}
+                theme_settings = p / "settings.json"
+                if theme_settings.is_file:
+                    try:
+                        with theme_settings.open() as f:
+                            settings = json.load(f)
+                    except Exception as e:
+                        log.warning(_(
+                            "Can't load theme settings at {path}: {e}").format(
+                            path=theme_settings, e=e))
+                    else:
+                        log.debug(
+                            f"found settings for theme {p.name!r} at {theme_settings}")
+                        fallback = settings.get("fallback")
+                        if fallback is None:
+                            settings["fallback"] = []
+                        elif isinstance(fallback, str):
+                            settings["fallback"] = [fallback]
+                        elif not isinstance(fallback, list):
+                            raise ValueError(
+                                'incorrect type for "fallback" in settings '
+                                f'({type(fallback)}) at {theme_settings}: {fallback}'
+                            )
+                        theme_data['settings'] = settings
+                browser_path = p / BROWSER_DIR
+                if browser_path.is_dir():
+                    theme_data['browser_path'] = browser_path
+                browser_meta_path = browser_path / BROWSER_META_FILE
+                if browser_meta_path.is_file():
+                    try:
+                        with browser_meta_path.open() as f:
+                            theme_data['browser_meta'] = json.load(f)
+                    except Exception as e:
+                        log.error(
+                            f"Can't parse browser metadata at {browser_meta_path}: {e}"
+                        )
+                        continue
+
+        self.env = Environment(
+            loader=TemplateLoader(
+                sites_paths=self.sites_paths,
+                sites_themes=self.sites_themes,
+                trusted=trusted
+            ),
+            autoescape=jinja2.select_autoescape(["html", "xhtml", "xml"]),
+            trim_blocks=True,
+            lstrip_blocks=True,
+            extensions=["jinja2.ext.i18n"],
+        )
+        self.env._template_data = None
+        self._locale_str = C.DEFAULT_LOCALE
+        self._locale = Locale.parse(self._locale_str)
+        self.install_translations()
+
+        # we want to have access to SàT constants in templates
+        self.env.globals["C"] = C
+
+        # custom filters
+        self.env.filters["next_gidx"] = self._next_gidx
+        self.env.filters["cur_gidx"] = self._cur_gidx
+        self.env.filters["date_fmt"] = self._date_fmt
+        self.env.filters["timestamp_to_hour"] = self._timestamp_to_hour
+        self.env.filters["delta_to_human"] = date_utils.delta2human
+        self.env.filters["xmlui_class"] = self._xmlui_class
+        self.env.filters["attr_escape"] = self.attr_escape
+        self.env.filters["item_filter"] = self._item_filter
+        self.env.filters["adv_format"] = self._adv_format
+        self.env.filters["dict_ext"] = self._dict_ext
+        self.env.filters["highlight"] = self.highlight
+        self.env.filters["front_url"] = (self._front_url if front_url_filter is None
+                                         else front_url_filter)
+        # custom tests
+        self.env.tests["in_the_past"] = self._in_the_past
+        self.icons_path = os.path.join(host.media_dir, "fonts/fontello/svg")
+
+        # policies
+        self.env.policies["ext.i18n.trimmed"] = True
+        self.env.policies["json.dumps_kwargs"] = {
+            "sort_keys": True,
+            # if object can't be serialised, we use None
+            "default": lambda o: o.to_json() if hasattr(o, "to_json") else None
+        }
+
+    def get_front_url(self, template_data, path=None):
+        """Give front URL (i.e. URL seen by end-user) of a path
+
+        @param template_data[TemplateData]: data of current template
+        @param path(unicode, None): relative path of file to get,
+            if set, will remplate template_data.path
+        """
+        return self.env.filters["front_url"]({"template_data": template_data},
+                                path or template_data.path)
+
+    def install_translations(self):
+        # TODO: support multi translation
+        #       for now, only translations in sat_templates are handled
+        self.translations = {}
+        for site_key, site_path in self.sites_paths.items():
+            site_prefix = "[{}] ".format(site_key) if site_key else ''
+            i18n_dir = os.path.join(site_path, "i18n")
+            for lang_dir in os.listdir(i18n_dir):
+                lang_path = os.path.join(i18n_dir, lang_dir)
+                if not os.path.isdir(lang_path):
+                    continue
+                po_path = os.path.join(lang_path, "LC_MESSAGES/sat.mo")
+                try:
+                    locale = Locale.parse(lang_dir)
+                    with open(po_path, "rb") as f:
+                        try:
+                            translations = self.translations[locale]
+                        except KeyError:
+                            self.translations[locale] = support.Translations(f, "sat")
+                        else:
+                            translations.merge(support.Translations(f, "sat"))
+                except EnvironmentError:
+                    log.error(
+                        _("Can't find template translation at {path}").format(
+                            path=po_path))
+                except UnknownLocaleError as e:
+                    log.error(_("{site}Invalid locale name: {msg}").format(
+                        site=site_prefix, msg=e))
+                else:
+                    log.info(_("{site}loaded {lang} templates translations").format(
+                        site = site_prefix,
+                        lang=lang_dir))
+
+        default_locale = Locale.parse(self._locale_str)
+        if default_locale not in self.translations:
+            # default locale disable gettext,
+            # so we can use None instead of a Translations instance
+            self.translations[default_locale] = None
+
+        self.env.install_null_translations(True)
+        # we generate a tuple of locales ordered by display name that templates can access
+        # through the "locales" variable
+        self.locales = tuple(sorted(list(self.translations.keys()),
+                                    key=lambda l: l.language_name.lower()))
+
+
+    def set_locale(self, locale_str):
+        """set current locale
+
+        change current translation locale and self self._locale and self._locale_str
+        """
+        if locale_str == self._locale_str:
+            return
+        if locale_str == "en":
+            # we default to GB English when it's not specified
+            # one of the main reason is to avoid the nonsense U.S. short date format
+            locale_str = "en_GB"
+        try:
+            locale = Locale.parse(locale_str)
+        except ValueError as e:
+            log.warning(_("invalid locale value: {msg}").format(msg=e))
+            locale_str = self._locale_str = C.DEFAULT_LOCALE
+            locale = Locale.parse(locale_str)
+
+        locale_str = str(locale)
+        if locale_str != C.DEFAULT_LOCALE:
+            try:
+                translations = self.translations[locale]
+            except KeyError:
+                log.warning(_("Can't find locale {locale}".format(locale=locale)))
+                locale_str = C.DEFAULT_LOCALE
+                locale = Locale.parse(self._locale_str)
+            else:
+                self.env.install_gettext_translations(translations, True)
+                log.debug(_("Switched to {lang}").format(lang=locale.english_name))
+
+        if locale_str == C.DEFAULT_LOCALE:
+            self.env.install_null_translations(True)
+
+        self._locale = locale
+        self._locale_str = locale_str
+
+    def get_theme_and_root(self, template):
+        """retrieve theme and root dir of a given template
+
+        @param template(unicode): template to parse
+        @return (tuple[unicode, unicode]): theme and absolute path to theme's root dir
+        @raise NotFound: requested site has not been found
+        """
+        # FIXME: check use in jp, and include site
+        site, theme, __ = self.env.loader.parse_template(template)
+        if site is None:
+            # absolute template
+            return  "", os.path.dirname(template)
+        try:
+            site_root_dir = self.sites_paths[site]
+        except KeyError:
+            raise exceptions.NotFound
+        return theme, os.path.join(site_root_dir, C.TEMPLATE_TPL_DIR, theme)
+
+    def get_themes_data(self, site_name):
+        try:
+            return self.sites_themes[site_name]
+        except KeyError:
+            raise exceptions.NotFound(f"no theme found for {site_name}")
+
+    def get_static_path(
+            self,
+            template_data: TemplateData,
+            filename: str,
+            settings: Optional[dict]=None
+        ) -> Optional[TemplateData]:
+        """Retrieve path of a static file if it exists with current theme or default
+
+        File will be looked at <site_root_dir>/<theme_dir>/<static_dir>/filename,
+        then <site_root_dir>/<default_theme_dir>/<static_dir>/filename anf finally
+        <default_site>/<default_theme_dir>/<static_dir> (i.e. sat_templates).
+        In case of absolute URL, base dir of template is used as base. For instance if
+        template is an absolute template to "/some/path/template.html", file will be
+        checked at "/some/path/<filename>"
+        @param template_data: data of current template
+        @param filename: name of the file to retrieve
+        @param settings: theme settings, can be used to modify behaviour
+        @return: built template data instance where .path is
+            the relative path to the file, from theme root dir.
+            None if not found.
+        """
+        if template_data.site is None:
+            # we have an absolue path
+            if (not template_data.theme is None
+                or not template_data.path.startswith('/')):
+                raise exceptions.InternalError(
+                    "invalid template data, was expecting absolute URL")
+            static_dir = os.path.dirname(template_data.path)
+            file_path = os.path.join(static_dir, filename)
+            if os.path.exists(file_path):
+                return TemplateData(site=None, theme=None, path=file_path)
+            else:
+                return None
+
+        sites_and_themes = TemplateLoader.get_sites_and_themes(template_data.site,
+                                                            template_data.theme,
+                                                            settings)
+        for site, theme in sites_and_themes:
+            site_root_dir = self.sites_paths[site]
+            relative_path = os.path.join(C.TEMPLATE_STATIC_DIR, filename)
+            absolute_path = os.path.join(site_root_dir, C.TEMPLATE_TPL_DIR,
+                                         theme, relative_path)
+            if os.path.exists(absolute_path):
+                return TemplateData(site=site, theme=theme, path=relative_path)
+
+        return None
+
+    def _append_css_paths(
+            self,
+            template_data: TemplateData,
+            css_files: list,
+            css_files_noscript: list,
+            name_root: str,
+            settings: dict
+
+        ) -> None:
+        """Append found css to css_files and css_files_noscript
+
+        @param css_files: list to fill of relative path to found css file
+        @param css_files_noscript: list to fill of relative path to found css file
+            with "_noscript" suffix
+        """
+        name = name_root + ".css"
+        css_path = self.get_static_path(template_data, name, settings)
+        if css_path is not None:
+            css_files.append(self.get_front_url(css_path))
+            noscript_name = name_root + "_noscript.css"
+            noscript_path = self.get_static_path(
+                template_data, noscript_name, settings)
+            if noscript_path is not None:
+                css_files_noscript.append(self.get_front_url(noscript_path))
+
+    def get_css_files(self, template_data):
+        """Retrieve CSS files to use according template_data
+
+        For each element of the path, a .css file is looked for in /static, and returned
+        if it exists.
+        Previous element are kept by replacing '/' with '_'.
+        styles_extra.css, styles.css, highlight.css and fonts.css are always used if they
+            exist.
+        For each found file, if a file with the same name and "_noscript" suffix exists,
+        it will be returned is second part of resulting tuple.
+        For instance, if template_data is (some_site, some_theme, blog/articles.html),
+        following files are returned, each time trying [some_site root] first,
+        then default site (i.e. sat_templates) root:
+            - some_theme/static/styles.css is returned if it exists
+              else default/static/styles.css
+            - some_theme/static/blog.css is returned if it exists
+              else default/static/blog.css (if it exists too)
+            - some_theme/static/blog_articles.css is returned if it exists
+              else default/static/blog_articles.css (if it exists too)
+        and for each found files, if same file with _noscript suffix exists, it is put
+        in noscript list (for instance (some_theme/static/styles_noscript.css)).
+        The behaviour may be changed with theme settings: if "fallback" is set, specified
+        themes will be checked instead of default. The theme will be checked in given
+        order, and "fallback" may be None or empty list to not check anything.
+        @param template_data(TemplateData): data of the current template
+        @return (tuple[list[unicode], list[unicode]]): a tuple with:
+            - front URLs of CSS files to use
+            - front URLs of CSS files to use when scripts are not enabled
+        """
+        # TODO: some caching would be nice
+        css_files = []
+        css_files_noscript = []
+        path_elems = template_data.path.split('/')
+        path_elems[-1] = os.path.splitext(path_elems[-1])[0]
+        site = template_data.site
+        if site is None:
+            # absolute path
+            settings = {}
+        else:
+            settings = self.sites_themes[site][template_data.theme]['settings']
+
+        css_path = self.get_static_path(template_data, 'fonts.css', settings)
+        if css_path is not None:
+            css_files.append(self.get_front_url(css_path))
+
+        for name_root in ('styles', 'styles_extra', 'highlight'):
+            self._append_css_paths(
+                template_data, css_files, css_files_noscript, name_root, settings)
+
+        for idx in range(len(path_elems)):
+            name_root = "_".join(path_elems[:idx+1])
+            self._append_css_paths(
+                template_data, css_files, css_files_noscript, name_root, settings)
+
+        return css_files, css_files_noscript
+
+    ## custom filters ##
+
+    @contextfilter
+    def _front_url(self, ctx, relative_url):
+        """Get front URL (URL seen by end-user) from a relative URL
+
+        This default method return absolute full path
+        """
+        template_data = ctx['template_data']
+        if template_data.site is None:
+            assert template_data.theme is None
+            assert template_data.path.startswith("/")
+            return os.path.join(os.path.dirname(template_data.path, relative_url))
+
+        site_root_dir = self.sites_paths[template_data.site]
+        return os.path.join(site_root_dir, C.TEMPLATE_TPL_DIR, template_data.theme,
+                            relative_url)
+
+    @contextfilter
+    def _next_gidx(self, ctx, value):
+        """Use next current global index as suffix"""
+        next_ = ctx["gidx"].next(value)
+        return value if next_ == 0 else "{}_{}".format(value, next_)
+
+    @contextfilter
+    def _cur_gidx(self, ctx, value):
+        """Use current current global index as suffix"""
+        current = ctx["gidx"].current(value)
+        return value if not current else "{}_{}".format(value, current)
+
+    def _date_fmt(
+        self,
+        timestamp: Union[int, float],
+        fmt: str = "short",
+        date_only: bool = False,
+        auto_limit: int = 7,
+        auto_old_fmt: str = "short",
+        auto_new_fmt: str = "relative",
+        tz_name: Optional[str] = None
+    ) -> str:
+        if is_undefined(fmt):
+            fmt = "short"
+
+        try:
+            return date_utils.date_fmt(
+                timestamp, fmt, date_only, auto_limit, auto_old_fmt,
+                auto_new_fmt, locale_str = self._locale_str,
+                tz_info=tz_name or date_utils.TZ_UTC
+            )
+        except Exception as e:
+            log.warning(_("Can't parse date: {msg}").format(msg=e))
+            return str(timestamp)
+
+    def _timestamp_to_hour(self, timestamp: float) -> int:
+        """Get hour of day corresponding to a timestamp"""
+        dt = datetime.fromtimestamp(timestamp)
+        return dt.hour
+
+    def attr_escape(self, text):
+        """escape a text to a value usable as an attribute
+
+        remove spaces, and put in lower case
+        """
+        return RE_ATTR_ESCAPE.sub("_", text.strip().lower())[:50]
+
+    def _xmlui_class(self, xmlui_item, fields):
+        """return classes computed from XMLUI fields name
+
+        will return a string with a series of escaped {name}_{value} separated by spaces.
+        @param xmlui_item(xmlui.XMLUIPanel): XMLUI containing the widgets to use
+        @param fields(iterable(unicode)): names of the widgets to use
+        @return (unicode, None): computer string to use as class attribute value
+            None if no field was specified
+        """
+        classes = []
+        for name in fields:
+            escaped_name = self.attr_escape(name)
+            try:
+                for value in xmlui_item.widgets[name].values:
+                    classes.append(escaped_name + "_" + self.attr_escape(value))
+            except KeyError:
+                log.debug(
+                    _('ignoring field "{name}": it doesn\'t exists').format(name=name)
+                )
+                continue
+        return " ".join(classes) or None
+
+    @contextfilter
+    def _item_filter(self, ctx, item, filters):
+        """return item's value, filtered if suitable
+
+        @param item(object): item to filter
+            value must have name and value attributes,
+            mostly used for XMLUI items
+        @param filters(dict[unicode, (callable, dict, None)]): map of name => filter
+            if filter is None, return the value unchanged
+            if filter is a callable, apply it
+            if filter is a dict, it can have following keys:
+                - filters: iterable of filters to apply
+                - filters_args: kwargs of filters in the same order as filters (use empty
+                                dict if needed)
+                - template: template to format where {value} is the filtered value
+        """
+        value = item.value
+        filter_ = filters.get(item.name, None)
+        if filter_ is None:
+            return value
+        elif isinstance(filter_, dict):
+            filters_args = filter_.get("filters_args")
+            for idx, f_name in enumerate(filter_.get("filters", [])):
+                kwargs = filters_args[idx] if filters_args is not None else {}
+                filter_func = self.env.filters[f_name]
+                try:
+                    eval_context_filter = filter_func.evalcontextfilter
+                except AttributeError:
+                    eval_context_filter = False
+
+                if eval_context_filter:
+                    value = filter_func(ctx.eval_ctx, value, **kwargs)
+                else:
+                    value = filter_func(value, **kwargs)
+            template = filter_.get("template")
+            if template:
+                # format will return a string, so we need to check first
+                # if the value is safe or not, and re-mark it after formatting
+                is_safe = isinstance(value, safe)
+                value = template.format(value=value)
+                if is_safe:
+                    value = safe(value)
+            return value
+
+    def _adv_format(self, value, template, **kwargs):
+        """Advancer formatter
+
+        like format() method, but take care or special values like None
+        @param value(unicode): value to format
+        @param template(None, unicode): template to use with format() method.
+            It will be formatted using value=value and **kwargs
+            None to return value unchanged
+        @return (unicode): formatted value
+        """
+        if template is None:
+            return value
+        #  jinja use string when no special char is used, so we have to convert to unicode
+        return str(template).format(value=value, **kwargs)
+
+    def _dict_ext(self, source_dict, extra_dict, key=None):
+        """extend source_dict with extra dict and return the result
+
+        @param source_dict(dict): dictionary to extend
+        @param extra_dict(dict, None): dictionary to use to extend first one
+            None to return source_dict unmodified
+        @param key(unicode, None): if specified extra_dict[key] will be used
+            if it doesn't exists, a copy of unmodified source_dict is returned
+        @return (dict): resulting dictionary
+        """
+        if extra_dict is None:
+            return source_dict
+        if key is not None:
+            extra_dict = extra_dict.get(key, {})
+        ret = source_dict.copy()
+        ret.update(extra_dict)
+        return ret
+
+    def highlight(self, code, lexer_name=None, lexer_opts=None, html_fmt_opts=None):
+        """Do syntax highlighting on code
+
+        Under the hood, pygments is used, check its documentation for options possible
+        values.
+        @param code(unicode): code or markup to highlight
+        @param lexer_name(unicode, None): name of the lexer to use
+            None to autodetect it
+        @param html_fmt_opts(dict, None): kword arguments to use for HtmlFormatter
+        @return (unicode): HTML markup with highlight classes
+        """
+        if lexer_opts is None:
+            lexer_opts = {}
+        if html_fmt_opts is None:
+            html_fmt_opts = {}
+        if lexer_name is None:
+            lexer = lexers.guess_lexer(code, **lexer_opts)
+        else:
+            lexer = lexers.get_lexer_by_name(lexer_name, **lexer_opts)
+        formatter = formatters.HtmlFormatter(**html_fmt_opts)
+        return safe(pygments.highlight(code, lexer, formatter))
+
+    ## custom tests ##
+
+    def _in_the_past(self, timestamp):
+        """check if a date is in the past
+
+        @param timestamp(unicode, int): unix time
+        @return (bool): True if date is in the past
+        """
+        return time.time() > int(timestamp)
+
+    ## template methods ##
+
+    def _icon_defs(self, *names):
+        """Define svg icons which will be used in the template.
+
+        Their name is used as id
+        """
+        svg_elt = etree.Element(
+            "svg",
+            nsmap={None: "http://www.w3.org/2000/svg"},
+            width="0",
+            height="0",
+            style="display: block",
+        )
+        defs_elt = etree.SubElement(svg_elt, "defs")
+        for name in names:
+            path = os.path.join(self.icons_path, name + ".svg")
+            icon_svg_elt = etree.parse(path).getroot()
+            # we use icon name as id, so we can retrieve them easily
+            icon_svg_elt.set("id", name)
+            if not icon_svg_elt.tag == "{http://www.w3.org/2000/svg}svg":
+                raise exceptions.DataError("invalid SVG element")
+            defs_elt.append(icon_svg_elt)
+        return safe(etree.tostring(svg_elt, encoding="unicode"))
+
+    def _icon_use(self, name, cls=""):
+        return safe('<svg class="svg-icon{cls}" xmlns="http://www.w3.org/2000/svg" '
+                    'viewBox="0 0 100 100">\n'
+                    '    <use href="#{name}"/>'
+                    '</svg>\n'.format(name=name, cls=(" " + cls) if cls else ""))
+
+    def _icon_from_client(self, client):
+        """Get icon name to represent a disco client"""
+        if client is None:
+            return 'desktop'
+        elif 'pc' in client:
+            return 'desktop'
+        elif 'phone' in client:
+            return 'mobile'
+        elif 'web' in client:
+            return 'globe'
+        elif 'console' in client:
+            return 'terminal'
+        else:
+            return 'desktop'
+
+    def render(self, template, site=None, theme=None, locale=C.DEFAULT_LOCALE,
+               media_path="", css_files=None, css_inline=False, **kwargs):
+        """Render a template
+
+        @param template(unicode): template to render (e.g. blog/articles.html)
+        @param site(unicode): site name
+            None or empty string for defaut site (i.e. SàT templates)
+        @param theme(unicode): template theme
+        @param media_path(unicode): prefix of the SàT media path/URL to use for
+            template root. Must end with a u'/'
+        @param css_files(list[unicode],None): CSS files to use
+            CSS files must be in static dir of the template
+            use None for automatic selection of CSS files based on template category
+            None is recommended. General static/style.css and theme file name will be
+            used.
+        @param css_inline(bool): if True, CSS will be embedded in the HTML page
+        @param **kwargs: variable to transmit to the template
+        """
+        if not template:
+            raise ValueError("template can't be empty")
+        if site is not None or theme is not None:
+            # user wants to set site and/or theme, so we add it to the template path
+            if site is None:
+                site = ''
+            if theme is None:
+                theme = C.TEMPLATE_THEME_DEFAULT
+            if template[0] == "(":
+                raise ValueError(
+                    "you can't specify site or theme in template path and in argument "
+                    "at the same time"
+                )
+
+            template_data = TemplateData(site, theme, template)
+            template = "({site}/{theme}){template}".format(
+                site=site, theme=theme, template=template)
+        else:
+            template_data = self.env.loader.parse_template(template)
+
+        # we need to save template_data in environment, to load right templates when they
+        # are referenced from other templates (e.g. import)
+        # FIXME: this trick will not work anymore if we use async templates (it works
+        #        here because we know that the rendering will be blocking until we unset
+        #        _template_data)
+        self.env._template_data = template_data
+
+        template_source = self.env.get_template(template)
+
+        if css_files is None:
+            css_files, css_files_noscript = self.get_css_files(template_data)
+        else:
+            css_files_noscript = []
+
+        kwargs["icon_defs"] = self._icon_defs
+        kwargs["icon"] = self._icon_use
+        kwargs["icon_from_client"] = self._icon_from_client
+
+        if css_inline:
+            css_contents = []
+            for files, suffix in ((css_files, ""),
+                                  (css_files_noscript, "_noscript")):
+                site_root_dir = self.sites_paths[template_data.site]
+                for css_file in files:
+                    css_file_path = os.path.join(site_root_dir, css_file)
+                    with open(css_file_path) as f:
+                        css_contents.append(f.read())
+                if css_contents:
+                    kwargs["css_content" + suffix] = "\n".join(css_contents)
+
+        scripts_handler = ScriptsHandler(self, template_data)
+        self.set_locale(locale)
+
+        # XXX: theme used in template arguments is the requested theme, which may differ
+        #      from actual theme if the template doesn't exist in the requested theme.
+        rendered = template_source.render(
+            template_data=template_data,
+            media_path=media_path,
+            css_files=css_files,
+            css_files_noscript=css_files_noscript,
+            locale=self._locale,
+            locales=self.locales,
+            gidx=Indexer(),
+            script=scripts_handler,
+            **kwargs
+        )
+        self.env._template_data = None
+        return rendered