#!/usr/bin/env python3

# Libervia: an XMPP client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from libervia.backend.core.i18n import _, D_

from libervia.backend.core import exceptions
from libervia.backend.core.constants import Const as C
from libervia.backend.memory.crypto import BlockCipher, PasswordHasher
from xml.dom import minidom, NotFoundErr
from libervia.backend.core.log import getLogger

log = getLogger(__name__)
from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.words.xish import domish
from twisted.words.protocols.jabber import jid
from libervia.backend.tools.xml_tools import params_xml_2_xmlui, get_text
from libervia.backend.tools.common import data_format
from xml.sax.saxutils import quoteattr

# TODO: params should be rewritten using Twisted directly instead of minidom
#       general params should be linked to sat.conf and kept synchronised
#       this need an overall simplification to make maintenance easier


def create_jid_elts(jids):
    """Generator which return <jid/> elements from jids

    @param jids(iterable[id.jID]): jids to use
    @return (generator[domish.Element]): <jid/> elements
    """
    for jid_ in jids:
        jid_elt = domish.Element((None, "jid"))
        jid_elt.addContent(jid_.full())
        yield jid_elt


class Params(object):
    """This class manage parameters with xml"""

    ### TODO: add desciption in params

    # TODO: when priority is changed, a new presence stanza must be emitted
    # TODO: int type (Priority should be int instead of string)
    default_xml = """
    <params>
    <general>
    </general>
    <individual>
        <category name="General" label="%(category_general)s">
            <param name="Password" value="" type="password" />
            <param name="%(history_param)s" label="%(history_label)s" value="20" constraint="0;100" type="int" security="0" />
            <param name="%(show_offline_contacts)s" label="%(show_offline_contacts_label)s" value="false" type="bool" security="0" />
            <param name="%(show_empty_groups)s" label="%(show_empty_groups_label)s" value="true" type="bool" security="0" />
        </category>
        <category name="Connection" label="%(category_connection)s">
            <param name="JabberID" value="name@example.org" type="string" security="10" />
            <param name="Password" value="" type="password" security="10" />
            <param name="Priority" value="50" type="int" constraint="-128;127" security="10" />
            <param name="%(force_server_param)s" value="" type="string" security="50" />
            <param name="%(force_port_param)s" value="" type="int" constraint="1;65535" security="50" />
            <param name="autoconnect_backend" label="%(autoconnect_backend_label)s" value="false" type="bool" security="50" />
            <param name="autoconnect" label="%(autoconnect_label)s" value="true" type="bool" security="50" />
            <param name="autodisconnect" label="%(autodisconnect_label)s" value="false"  type="bool" security="50" />
            <param name="check_certificate" label="%(check_certificate_label)s" value="true"  type="bool" security="4" />
        </category>
    </individual>
    </params>
    """ % {
        "category_general": D_("General"),
        "category_connection": D_("Connection"),
        "history_param": C.HISTORY_LIMIT,
        "history_label": D_("Chat history limit"),
        "show_offline_contacts": C.SHOW_OFFLINE_CONTACTS,
        "show_offline_contacts_label": D_("Show offline contacts"),
        "show_empty_groups": C.SHOW_EMPTY_GROUPS,
        "show_empty_groups_label": D_("Show empty groups"),
        "force_server_param": C.FORCE_SERVER_PARAM,
        "force_port_param": C.FORCE_PORT_PARAM,
        "autoconnect_backend_label": D_("Connect on backend startup"),
        "autoconnect_label": D_("Connect on frontend startup"),
        "autodisconnect_label": D_("Disconnect on frontend closure"),
        "check_certificate_label": D_("Check certificate (don't uncheck if unsure)"),
    }

    def load_default_params(self):
        self.dom = minidom.parseString(Params.default_xml.encode("utf-8"))

    def _merge_params(self, source_node, dest_node):
        """Look for every node in source_node and recursively copy them to dest if they don't exists"""

        def get_nodes_map(children):
            ret = {}
            for child in children:
                if child.nodeType == child.ELEMENT_NODE:
                    ret[(child.tagName, child.getAttribute("name"))] = child
            return ret

        source_map = get_nodes_map(source_node.childNodes)
        dest_map = get_nodes_map(dest_node.childNodes)
        source_set = set(source_map.keys())
        dest_set = set(dest_map.keys())
        to_add = source_set.difference(dest_set)

        for node_key in to_add:
            dest_node.appendChild(source_map[node_key].cloneNode(True))

        to_recurse = source_set - to_add
        for node_key in to_recurse:
            self._merge_params(source_map[node_key], dest_map[node_key])

    def load_xml(self, xml_file):
        """Load parameters template from xml file"""
        self.dom = minidom.parse(xml_file)
        default_dom = minidom.parseString(Params.default_xml.encode("utf-8"))
        self._merge_params(default_dom.documentElement, self.dom.documentElement)

    def load_gen_params(self):
        """Load general parameters data from storage

        @return: deferred triggered once params are loaded
        """
        return self.storage.load_gen_params(self.params_gen)

    def load_ind_params(self, profile, cache=None):
        """Load individual parameters

        set self.params cache or a temporary cache
        @param profile: profile to load (*must exist*)
        @param cache: if not None, will be used to store the value, as a short time cache
        @return: deferred triggered once params are loaded
        """
        if cache is None:
            self.params[profile] = {}
        return self.storage.load_ind_params(
            self.params[profile] if cache is None else cache, profile
        )

    def purge_profile(self, profile):
        """Remove cache data of a profile

        @param profile: %(doc_profile)s
        """
        try:
            del self.params[profile]
        except KeyError:
            log.error(
                _("Trying to purge cache of a profile not in memory: [%s]") % profile
            )

    def save_xml(self, filename):
        """Save parameters template to xml file"""
        with open(filename, "wb") as xml_file:
            xml_file.write(self.dom.toxml("utf-8"))

    def __init__(self, host, storage):
        log.debug("Parameters init")
        self.host = host
        self.storage = storage
        self.default_profile = None
        self.params = {}
        self.params_gen = {}

    def create_profile(self, profile, component):
        """Create a new profile

        @param profile(unicode): name of the profile
        @param component(unicode): entry point if profile is a component
        @param callback: called when the profile actually exists in database and memory
        @return: a Deferred instance
        """
        if self.storage.has_profile(profile):
            log.info(_("The profile name already exists"))
            return defer.fail(exceptions.ConflictError())
        if not self.host.trigger.point("ProfileCreation", profile):
            return defer.fail(exceptions.CancelError())
        return self.storage.create_profile(profile, component or None)

    def profile_delete_async(self, profile, force=False):
        """Delete an existing profile

        @param profile: name of the profile
        @param force: force the deletion even if the profile is connected.
        To be used for direct calls only (not through the bridge).
        @return: a Deferred instance
        """
        if not self.storage.has_profile(profile):
            log.info(_("Trying to delete an unknown profile"))
            return defer.fail(Failure(exceptions.ProfileUnknownError(profile)))
        if self.host.is_connected(profile):
            if force:
                self.host.disconnect(profile)
            else:
                log.info(_("Trying to delete a connected profile"))
                return defer.fail(Failure(exceptions.ProfileConnected))
        return self.storage.delete_profile(profile)

    def get_profile_name(self, profile_key, return_profile_keys=False):
        """return profile according to profile_key

        @param profile_key: profile name or key which can be
                            C.PROF_KEY_ALL for all profiles
                            C.PROF_KEY_DEFAULT for default profile
        @param return_profile_keys: if True, return unmanaged profile keys (like
            C.PROF_KEY_ALL). This keys must be managed by the caller
        @return: requested profile name
        @raise exceptions.ProfileUnknownError: profile doesn't exists
        @raise exceptions.ProfileNotSetError: if C.PROF_KEY_NONE is used
        """
        if profile_key == "@DEFAULT@":
            default = self.host.memory.memory_data.get("Profile_default")
            if not default:
                log.info(_("No default profile, returning first one"))
                try:
                    default = self.host.memory.memory_data[
                        "Profile_default"
                    ] = self.storage.get_profiles_list()[0]
                except IndexError:
                    log.info(_("No profile exist yet"))
                    raise exceptions.ProfileUnknownError(profile_key)
            return (
                default
            )  # FIXME: temporary, must use real default value, and fallback to first one if it doesn't exists
        elif profile_key == C.PROF_KEY_NONE:
            raise exceptions.ProfileNotSetError
        elif return_profile_keys and profile_key in [C.PROF_KEY_ALL]:
            return profile_key  # this value must be managed by the caller
        if not self.storage.has_profile(profile_key):
            log.error(_("Trying to access an unknown profile (%s)") % profile_key)
            raise exceptions.ProfileUnknownError(profile_key)
        return profile_key

    def __get_unique_node(self, parent, tag, name):
        """return node with given tag

        @param parent: parent of nodes to check (e.g. documentElement)
        @param tag: tag to check (e.g. "category")
        @param name: name to check (e.g. "JID")
        @return: node if it exist or None
        """
        for node in parent.childNodes:
            if node.nodeName == tag and node.getAttribute("name") == name:
                # the node already exists
                return node
        # the node is new
        return None

    def update_params(self, xml, security_limit=C.NO_SECURITY_LIMIT, app=""):
        """import xml in parameters, update if the param already exists

        If security_limit is specified and greater than -1, the parameters
        that have a security level greater than security_limit are skipped.
        @param xml: parameters in xml form
        @param security_limit: -1 means no security, 0 is the maximum security then the higher the less secure
        @param app: name of the frontend registering the parameters or empty value
        """
        # TODO: should word with domish.Element
        src_parent = minidom.parseString(xml.encode("utf-8")).documentElement

        def pre_process_app_node(src_parent, security_limit, app):
            """Parameters that are registered from a frontend must be checked"""
            to_remove = []
            for type_node in src_parent.childNodes:
                if type_node.nodeName != C.INDIVIDUAL:
                    to_remove.append(type_node)  # accept individual parameters only
                    continue
                for cat_node in type_node.childNodes:
                    if cat_node.nodeName != "category":
                        to_remove.append(cat_node)
                        continue
                    to_remove_count = (
                        0
                    )  # count the params to be removed from current category
                    for node in cat_node.childNodes:
                        if node.nodeName != "param" or not self.check_security_limit(
                            node, security_limit
                        ):
                            to_remove.append(node)
                            to_remove_count += 1
                            continue
                        node.setAttribute("app", app)
                    if (
                        len(cat_node.childNodes) == to_remove_count
                    ):  # remove empty category
                        for __ in range(0, to_remove_count):
                            to_remove.pop()
                        to_remove.append(cat_node)
            for node in to_remove:
                node.parentNode.removeChild(node)

        def import_node(tgt_parent, src_parent):
            for child in src_parent.childNodes:
                if child.nodeName == "#text":
                    continue
                node = self.__get_unique_node(
                    tgt_parent, child.nodeName, child.getAttribute("name")
                )
                if not node:  # The node is new
                    tgt_parent.appendChild(child.cloneNode(True))
                else:
                    if child.nodeName == "param":
                        # The child updates an existing parameter, we replace the node
                        tgt_parent.replaceChild(child, node)
                    else:
                        # the node already exists, we recurse 1 more level
                        import_node(node, child)

        if app:
            pre_process_app_node(src_parent, security_limit, app)
        import_node(self.dom.documentElement, src_parent)

    def params_register_app(self, xml, security_limit, app):
        """Register frontend's specific parameters

        If security_limit is specified and greater than -1, the parameters
        that have a security level greater than security_limit are skipped.
        @param xml: XML definition of the parameters to be added
        @param security_limit: -1 means no security, 0 is the maximum security then the higher the less secure
        @param app: name of the frontend registering the parameters
        """
        if not app:
            log.warning(
                _(
                    "Trying to register frontends parameters with no specified app: aborted"
                )
            )
            return
        if not hasattr(self, "frontends_cache"):
            self.frontends_cache = []
        if app in self.frontends_cache:
            log.debug(
                _(
                    "Trying to register twice frontends parameters for %(app)s: aborted"
                    % {"app": app}
                )
            )
            return
        self.frontends_cache.append(app)
        self.update_params(xml, security_limit, app)
        log.debug("Frontends parameters registered for %(app)s" % {"app": app})

    def __default_ok(self, value, name, category):
        # FIXME: will not work with individual parameters
        self.param_set(name, value, category)

    def __default_ko(self, failure, name, category):
        log.error(
            _("Can't determine default value for [%(category)s/%(name)s]: %(reason)s")
            % {"category": category, "name": name, "reason": str(failure.value)}
        )

    def set_default(self, name, category, callback, errback=None):
        """Set default value of parameter

        'default_cb' attibute of parameter must be set to 'yes'
        @param name: name of the parameter
        @param category: category of the parameter
        @param callback: must return a string with the value (use deferred if needed)
        @param errback: must manage the error with args failure, name, category
        """
        # TODO: send signal param update if value changed
        # TODO: manage individual paramaters
        log.debug(
            "set_default called for %(category)s/%(name)s"
            % {"category": category, "name": name}
        )
        node = self._get_param_node(name, category, "@ALL@")
        if not node:
            log.error(
                _(
                    "Requested param [%(name)s] in category [%(category)s] doesn't exist !"
                )
                % {"name": name, "category": category}
            )
            return
        if node[1].getAttribute("default_cb") == "yes":
            # del node[1].attributes['default_cb'] # default_cb is not used anymore as a flag to know if we have to set the default value,
            # and we can still use it later e.g. to call a generic set_default method
            value = self._get_param(category, name, C.GENERAL)
            if value is None:  # no value set by the user: we have the default value
                log.debug("Default value to set, using callback")
                d = defer.maybeDeferred(callback)
                d.addCallback(self.__default_ok, name, category)
                d.addErrback(errback or self.__default_ko, name, category)

    def _get_attr_internal(self, node, attr, value):
        """Get attribute value.

        /!\ This method would return encrypted password values.

        @param node: XML param node
        @param attr: name of the attribute to get (e.g.: 'value' or 'type')
        @param value: user defined value
        @return: value (can be str, bool, int, list, None)
        """
        if attr == "value":
            value_to_use = (
                value if value is not None else node.getAttribute(attr)
            )  # we use value (user defined) if it exist, else we use node's default value
            if node.getAttribute("type") == "bool":
                return C.bool(value_to_use)
            if node.getAttribute("type") == "int":
                return int(value_to_use) if value_to_use else value_to_use
            elif node.getAttribute("type") == "list":
                if (
                    not value_to_use
                ):  # no user defined value, take default value from the XML
                    options = [
                        option
                        for option in node.childNodes
                        if option.nodeName == "option"
                    ]
                    selected = [
                        option
                        for option in options
                        if option.getAttribute("selected") == "true"
                    ]
                    cat, param = (
                        node.parentNode.getAttribute("name"),
                        node.getAttribute("name"),
                    )
                    if len(selected) == 1:
                        value_to_use = selected[0].getAttribute("value")
                        log.info(
                            _(
                                "Unset parameter (%(cat)s, %(param)s) of type list will use the default option '%(value)s'"
                            )
                            % {"cat": cat, "param": param, "value": value_to_use}
                        )
                        return value_to_use
                    if len(selected) == 0:
                        log.error(
                            _(
                                "Parameter (%(cat)s, %(param)s) of type list has no default option!"
                            )
                            % {"cat": cat, "param": param}
                        )
                    else:
                        log.error(
                            _(
                                "Parameter (%(cat)s, %(param)s) of type list has more than one default option!"
                            )
                            % {"cat": cat, "param": param}
                        )
                    raise exceptions.DataError
            elif node.getAttribute("type") == "jids_list":
                if value_to_use:
                    jids = value_to_use.split(
                        "\t"
                    )  # FIXME: it's not good to use tabs as separator !
                else:  # no user defined value, take default value from the XML
                    jids = [get_text(jid_) for jid_ in node.getElementsByTagName("jid")]
                to_delete = []
                for idx, value in enumerate(jids):
                    try:
                        jids[idx] = jid.JID(value)
                    except (RuntimeError, jid.InvalidFormat, AttributeError):
                        log.warning(
                            "Incorrect jid value found in jids list: [{}]".format(value)
                        )
                        to_delete.append(value)
                for value in to_delete:
                    jids.remove(value)
                return jids
            return value_to_use
        return node.getAttribute(attr)

    def _get_attr(self, node, attr, value):
        """Get attribute value (synchronous).

        /!\ This method can not be used to retrieve password values.
        @param node: XML param node
        @param attr: name of the attribute to get (e.g.: 'value' or 'type')
        @param value: user defined value
        @return (unicode, bool, int, list): value to retrieve
        """
        if attr == "value" and node.getAttribute("type") == "password":
            raise exceptions.InternalError(
                "To retrieve password values, use _async_get_attr instead of _get_attr"
            )
        return self._get_attr_internal(node, attr, value)

    def _async_get_attr(self, node, attr, value, profile=None):
        """Get attribute value.

        Profile passwords are returned hashed (if not empty),
        other passwords are returned decrypted (if not empty).
        @param node: XML param node
        @param attr: name of the attribute to get (e.g.: 'value' or 'type')
        @param value: user defined value
        @param profile: %(doc_profile)s
        @return (unicode, bool, int, list): Deferred value to retrieve
        """
        value = self._get_attr_internal(node, attr, value)
        if attr != "value" or node.getAttribute("type") != "password":
            return defer.succeed(value)
        param_cat = node.parentNode.getAttribute("name")
        param_name = node.getAttribute("name")
        if ((param_cat, param_name) == C.PROFILE_PASS_PATH) or not value:
            return defer.succeed(
                value
            )  # profile password and empty passwords are returned "as is"
        if not profile:
            raise exceptions.ProfileNotSetError(
                "The profile is needed to decrypt a password"
            )
        password = self.host.memory.decrypt_value(value, profile)

        if password is None:
            raise exceptions.InternalError("password should never be None")
        return defer.succeed(password)

    def _type_to_str(self, result):
        """Convert result to string, according to its type """
        if isinstance(result, bool):
            return C.bool_const(result)
        elif isinstance(result, (list, set, tuple)):
            return ', '.join(self._type_to_str(r) for r in result)
        else:
            return str(result)

    def get_string_param_a(self, name, category, attr="value", profile_key=C.PROF_KEY_NONE):
        """ Same as param_get_a but for bridge: convert non string value to string """
        return self._type_to_str(
            self.param_get_a(name, category, attr, profile_key=profile_key)
        )

    def param_get_a(
        self, name, category, attr="value", use_default=True, profile_key=C.PROF_KEY_NONE
    ):
        """Helper method to get a specific attribute.

        /!\ This method would return encrypted password values,
            to get the plain values you have to use param_get_a_async.
        @param name: name of the parameter
        @param category: category of the parameter
        @param attr: name of the attribute (default: "value")
        @parm use_default(bool): if True and attr=='value', return default value if not set
            else return None if not set
        @param profile: owner of the param (@ALL@ for everyone)
        @return: attribute
        """
        # FIXME: looks really dirty and buggy, need to be reviewed/refactored
        # FIXME: security_limit is not managed here !
        node = self._get_param_node(name, category)
        if not node:
            log.error(
                _(
                    "Requested param [%(name)s] in category [%(category)s] doesn't exist !"
                )
                % {"name": name, "category": category}
            )
            raise exceptions.NotFound

        if attr == "value" and node[1].getAttribute("type") == "password":
            raise exceptions.InternalError(
                "To retrieve password values, use param_get_a_async instead of param_get_a"
            )

        if node[0] == C.GENERAL:
            value = self._get_param(category, name, C.GENERAL)
            if value is None and attr == "value" and not use_default:
                return value
            return self._get_attr(node[1], attr, value)

        assert node[0] == C.INDIVIDUAL

        profile = self.get_profile_name(profile_key)
        if not profile:
            log.error(_("Requesting a param for an non-existant profile"))
            raise exceptions.ProfileUnknownError(profile_key)

        if profile not in self.params:
            log.error(_("Requesting synchronous param for not connected profile"))
            raise exceptions.ProfileNotConnected(profile)

        if attr == "value":
            value = self._get_param(category, name, profile=profile)
            if value is None and attr == "value" and not use_default:
                return value
            return self._get_attr(node[1], attr, value)

    async def async_get_string_param_a(
        self, name, category, attr="value", security_limit=C.NO_SECURITY_LIMIT,
        profile=C.PROF_KEY_NONE):
        value = await self.param_get_a_async(
            name, category, attr, security_limit, profile_key=profile)
        return self._type_to_str(value)

    def param_get_a_async(
        self,
        name,
        category,
        attr="value",
        security_limit=C.NO_SECURITY_LIMIT,
        profile_key=C.PROF_KEY_NONE,
    ):
        """Helper method to get a specific attribute.

        @param name: name of the parameter
        @param category: category of the parameter
        @param attr: name of the attribute (default: "value")
        @param profile: owner of the param (@ALL@ for everyone)
        @return (defer.Deferred): parameter value, with corresponding type (bool, int, list, etc)
        """
        node = self._get_param_node(name, category)
        if not node:
            log.error(
                _(
                    "Requested param [%(name)s] in category [%(category)s] doesn't exist !"
                )
                % {"name": name, "category": category}
            )
            raise ValueError("Requested param doesn't exist")

        if not self.check_security_limit(node[1], security_limit):
            log.warning(
                _(
                    "Trying to get parameter '%(param)s' in category '%(cat)s' without authorization!!!"
                    % {"param": name, "cat": category}
                )
            )
            raise exceptions.PermissionError

        if node[0] == C.GENERAL:
            value = self._get_param(category, name, C.GENERAL)
            return self._async_get_attr(node[1], attr, value)

        assert node[0] == C.INDIVIDUAL

        profile = self.get_profile_name(profile_key)
        if not profile:
            raise exceptions.InternalError(
                _("Requesting a param for a non-existant profile")
            )

        if attr != "value":
            return defer.succeed(node[1].getAttribute(attr))
        try:
            value = self._get_param(category, name, profile=profile)
            return self._async_get_attr(node[1], attr, value, profile)
        except exceptions.ProfileNotInCacheError:
            # We have to ask data to the storage manager
            d = self.storage.get_ind_param(category, name, profile)
            return d.addCallback(
                lambda value: self._async_get_attr(node[1], attr, value, profile)
            )

    def _get_params_values_from_category(
        self, category, security_limit, app, extra_s, profile_key):
        client = self.host.get_client(profile_key)
        extra = data_format.deserialise(extra_s)
        return defer.ensureDeferred(self.get_params_values_from_category(
            client, category, security_limit, app, extra))

    async def get_params_values_from_category(
        self, client, category, security_limit, app='', extra=None):
        """Get all parameters "attribute" for a category

        @param category(unicode): the desired category
        @param security_limit(int): NO_SECURITY_LIMIT (-1) to return all the params.
            Otherwise sole the params which have a security level defined *and*
            lower or equal to the specified value are returned.
        @param app(str): see [get_params]
        @param extra(dict): see [get_params]
        @return (dict): key: param name, value: param value (converted to string if needed)
        """
        # TODO: manage category of general type (without existant profile)
        if extra is None:
            extra = {}
        prof_xml = await self._construct_profile_xml(client, security_limit, app, extra)
        ret = {}
        for category_node in prof_xml.getElementsByTagName("category"):
            if category_node.getAttribute("name") == category:
                for param_node in category_node.getElementsByTagName("param"):
                    name = param_node.getAttribute("name")
                    if not name:
                        log.warning(
                            "ignoring attribute without name: {}".format(
                                param_node.toxml()
                            )
                        )
                        continue
                    value = await self.async_get_string_param_a(
                        name, category, security_limit=security_limit,
                        profile=client.profile)

                    ret[name] = value
                break

        prof_xml.unlink()
        return ret

    def _get_param(
        self, category, name, type_=C.INDIVIDUAL, cache=None, profile=C.PROF_KEY_NONE
    ):
        """Return the param, or None if it doesn't exist

        @param category: param category
        @param name: param name
        @param type_: GENERAL or INDIVIDUAL
        @param cache: temporary cache, to use when profile is not logged
        @param profile: the profile name (not profile key, i.e. name and not something like @DEFAULT@)
        @return: param value or None if it doesn't exist
        """
        if type_ == C.GENERAL:
            if (category, name) in self.params_gen:
                return self.params_gen[(category, name)]
            return None  # This general param has the default value
        assert type_ == C.INDIVIDUAL
        if profile == C.PROF_KEY_NONE:
            raise exceptions.ProfileNotSetError
        if profile in self.params:
            cache = self.params[profile]  # if profile is in main cache, we use it,
            # ignoring the temporary cache
        elif (
            cache is None
        ):  # else we use the temporary cache if it exists, or raise an exception
            raise exceptions.ProfileNotInCacheError
        if (category, name) not in cache:
            return None
        return cache[(category, name)]

    async def _construct_profile_xml(self, client, security_limit, app, extra):
        """Construct xml for asked profile, filling values when needed

        /!\ as noticed in doc, don't forget to unlink the minidom.Document
        @param security_limit: NO_SECURITY_LIMIT (-1) to return all the params.
        Otherwise sole the params which have a security level defined *and*
        lower or equal to the specified value are returned.
        @param app: name of the frontend requesting the parameters, or '' to get all parameters
        @param profile: profile name (not key !)
        @return: a deferred that fire a minidom.Document of the profile xml (cf warning above)
        """
        profile = client.profile

        def check_node(node):
            """Check the node against security_limit, app and extra"""
            return (self.check_security_limit(node, security_limit)
                    and self.check_app(node, app)
                    and self.check_extra(node, extra))

        if profile in self.params:
            profile_cache = self.params[profile]
        else:
            # profile is not in cache, we load values in a short time cache
            profile_cache = {}
            await self.load_ind_params(profile, profile_cache)

        # init the result document
        prof_xml = minidom.parseString("<params/>")
        cache = {}

        for type_node in self.dom.documentElement.childNodes:
            if type_node.nodeName != C.GENERAL and type_node.nodeName != C.INDIVIDUAL:
                continue
            # we use all params, general and individual
            for cat_node in type_node.childNodes:
                if cat_node.nodeName != "category":
                    continue
                category = cat_node.getAttribute("name")
                dest_params = {}  # result (merged) params for category
                if category not in cache:
                    # we make a copy for the new xml
                    cache[category] = dest_cat = cat_node.cloneNode(True)
                    to_remove = []
                    for node in dest_cat.childNodes:
                        if node.nodeName != "param":
                            continue
                        if not check_node(node):
                            to_remove.append(node)
                            continue
                        dest_params[node.getAttribute("name")] = node
                    for node in to_remove:
                        dest_cat.removeChild(node)
                    new_node = True
                else:
                    # It's not a new node, we use the previously cloned one
                    dest_cat = cache[category]
                    new_node = False
                params = cat_node.getElementsByTagName("param")

                for param_node in params:
                    # we have to merge new params (we are parsing individual parameters, we have to add them
                    # to the previously parsed general ones)
                    name = param_node.getAttribute("name")
                    if not check_node(param_node):
                        continue
                    if name not in dest_params:
                        # this is reached when a previous category exists
                        dest_params[name] = param_node.cloneNode(True)
                        dest_cat.appendChild(dest_params[name])

                    profile_value = self._get_param(
                        category,
                        name,
                        type_node.nodeName,
                        cache=profile_cache,
                        profile=profile,
                    )
                    if profile_value is not None:
                        # there is a value for this profile, we must change the default
                        if dest_params[name].getAttribute("type") == "list":
                            for option in dest_params[name].getElementsByTagName(
                                "option"
                            ):
                                if option.getAttribute("value") == profile_value:
                                    option.setAttribute("selected", "true")
                                else:
                                    try:
                                        option.removeAttribute("selected")
                                    except NotFoundErr:
                                        pass
                        elif dest_params[name].getAttribute("type") == "jids_list":
                            jids = profile_value.split("\t")
                            for jid_elt in dest_params[name].getElementsByTagName(
                                "jid"
                            ):
                                dest_params[name].removeChild(
                                    jid_elt
                                )  # remove all default
                            for jid_ in jids:  # rebuilt the children with use values
                                try:
                                    jid.JID(jid_)
                                except (
                                    RuntimeError,
                                    jid.InvalidFormat,
                                    AttributeError,
                                ):
                                    log.warning(
                                        "Incorrect jid value found in jids list: [{}]".format(
                                            jid_
                                        )
                                    )
                                else:
                                    jid_elt = prof_xml.createElement("jid")
                                    jid_elt.appendChild(prof_xml.createTextNode(jid_))
                                    dest_params[name].appendChild(jid_elt)
                        else:
                            dest_params[name].setAttribute("value", profile_value)
                if new_node:
                    prof_xml.documentElement.appendChild(dest_cat)

        to_remove = []
        for cat_node in prof_xml.documentElement.childNodes:
            # we remove empty categories
            if cat_node.getElementsByTagName("param").length == 0:
                to_remove.append(cat_node)
        for node in to_remove:
            prof_xml.documentElement.removeChild(node)

        return prof_xml


    def _get_params_ui(self, security_limit, app, extra_s, profile_key):
        client = self.host.get_client(profile_key)
        extra = data_format.deserialise(extra_s)
        return defer.ensureDeferred(self.param_ui_get(client, security_limit, app, extra))

    async def param_ui_get(self, client, security_limit, app, extra=None):
        """Get XMLUI to handle parameters

        @param security_limit: NO_SECURITY_LIMIT (-1) to return all the params.
            Otherwise sole the params which have a security level defined *and*
            lower or equal to the specified value are returned.
        @param app: name of the frontend requesting the parameters, or '' to get all parameters
        @param extra (dict, None): extra options. Key can be:
            - ignore: list of (category/name) values to remove from parameters
        @return(str): a SàT XMLUI for parameters
        """
        param_xml = await self.get_params(client, security_limit, app, extra)
        return params_xml_2_xmlui(param_xml)

    async def get_params(self, client, security_limit, app, extra=None):
        """Construct xml for asked profile, take params xml as skeleton

        @param security_limit: NO_SECURITY_LIMIT (-1) to return all the params.
            Otherwise sole the params which have a security level defined *and*
            lower or equal to the specified value are returned.
        @param app: name of the frontend requesting the parameters, or '' to get all parameters
        @param extra (dict, None): extra options. Key can be:
            - ignore: list of (category/name) values to remove from parameters
        @param profile_key: Profile key which can be either a magic (eg: @DEFAULT@) or the name of an existing profile.
        @return: XML of parameters
        """
        if extra is None:
            extra = {}
        prof_xml = await self._construct_profile_xml(client, security_limit, app, extra)
        return_xml = prof_xml.toxml()
        prof_xml.unlink()
        return "\n".join((line for line in return_xml.split("\n") if line))

    def _get_param_node(self, name, category, type_="@ALL@"):  # FIXME: is type_ useful ?
        """Return a node from the param_xml
        @param name: name of the node
        @param category: category of the node
        @param type_: keyword for search:
                                    @ALL@ search everywhere
                                    @GENERAL@ only search in general type
                                    @INDIVIDUAL@ only search in individual type
        @return: a tuple (node type, node) or None if not found"""

        for type_node in self.dom.documentElement.childNodes:
            if (
                (type_ == "@ALL@" or type_ == "@GENERAL@")
                and type_node.nodeName == C.GENERAL
            ) or (
                (type_ == "@ALL@" or type_ == "@INDIVIDUAL@")
                and type_node.nodeName == C.INDIVIDUAL
            ):
                for node in type_node.getElementsByTagName("category"):
                    if node.getAttribute("name") == category:
                        params = node.getElementsByTagName("param")
                        for param in params:
                            if param.getAttribute("name") == name:
                                return (type_node.nodeName, param)
        return None

    def params_categories_get(self):
        """return the categories availables"""
        categories = []
        for cat in self.dom.getElementsByTagName("category"):
            name = cat.getAttribute("name")
            if name not in categories:
                categories.append(cat.getAttribute("name"))
        return categories

    def param_set(self, name, value, category, security_limit=C.NO_SECURITY_LIMIT,
                 profile_key=C.PROF_KEY_NONE):
        """Set a parameter, return None if the parameter is not in param xml.

        Parameter of type 'password' that are not the SàT profile password are
        stored encrypted (if not empty). The profile password is stored hashed
        (if not empty).

        @param name (str): the parameter name
        @param value (str): the new value
        @param category (str): the parameter category
        @param security_limit (int)
        @param profile_key (str): %(doc_profile_key)s
        @return: a deferred None value when everything is done
        """
        # FIXME: param_set should accept the right type for value, not only str !
        if profile_key != C.PROF_KEY_NONE:
            profile = self.get_profile_name(profile_key)
            if not profile:
                log.error(_("Trying to set parameter for an unknown profile"))
                raise exceptions.ProfileUnknownError(profile_key)

        node = self._get_param_node(name, category, "@ALL@")
        if not node:
            log.error(
                _("Requesting an unknown parameter (%(category)s/%(name)s)")
                % {"category": category, "name": name}
            )
            return defer.succeed(None)

        if not self.check_security_limit(node[1], security_limit):
            msg = _(
                "{profile!r} is trying to set parameter {name!r} in category "
                "{category!r} without authorization!!!").format(
                    profile=repr(profile),
                    name=repr(name),
                    category=repr(category)
                )
            log.warning(msg)
            raise exceptions.PermissionError(msg)

        type_ = node[1].getAttribute("type")
        if type_ == "int":
            if not value:  # replace with the default value (which might also be '')
                value = node[1].getAttribute("value")
            else:
                try:
                    int(value)
                except ValueError:
                    log.warning(_(
                        "Trying to set parameter {name} in category {category} with"
                        "an non-integer value"
                    ).format(
                        name=repr(name),
                        category=repr(category)
                    ))
                    return defer.succeed(None)
                if node[1].hasAttribute("constraint"):
                    constraint = node[1].getAttribute("constraint")
                    try:
                        min_, max_ = [int(limit) for limit in constraint.split(";")]
                    except ValueError:
                        raise exceptions.InternalError(
                            "Invalid integer parameter constraint: %s" % constraint
                        )
                    value = str(min(max(int(value), min_), max_))

        log.info(
            _("Setting parameter (%(category)s, %(name)s) = %(value)s")
            % {
                "category": category,
                "name": name,
                "value": value if type_ != "password" else "********",
            }
        )

        if node[0] == C.GENERAL:
            self.params_gen[(category, name)] = value
            self.storage.set_gen_param(category, name, value)
            for profile in self.storage.get_profiles_list():
                if self.host.memory.is_session_started(profile):
                    self.host.bridge.param_update(name, value, category, profile)
                    self.host.trigger.point(
                        "param_update_trigger", name, value, category, node[0], profile
                    )
            return defer.succeed(None)

        assert node[0] == C.INDIVIDUAL
        assert profile_key != C.PROF_KEY_NONE

        if type_ == "button":
            log.debug("Clicked param button %s" % node.toxml())
            return defer.succeed(None)
        elif type_ == "password":
            try:
                personal_key = self.host.memory.auth_sessions.profile_get_unique(profile)[
                    C.MEMORY_CRYPTO_KEY
                ]
            except TypeError:
                raise exceptions.InternalError(
                    _("Trying to encrypt a password while the personal key is undefined!")
                )
            if (category, name) == C.PROFILE_PASS_PATH:
                # using 'value' as the encryption key to encrypt another encryption key... could be confusing!
                d = self.host.memory.encrypt_personal_data(
                    data_key=C.MEMORY_CRYPTO_KEY,
                    data_value=personal_key,
                    crypto_key=value,
                    profile=profile,
                )
                d.addCallback(
                    lambda __: PasswordHasher.hash(value)
                )  # profile password is hashed (empty value stays empty)
            elif value:  # other non empty passwords are encrypted with the personal key
                d = defer.succeed(BlockCipher.encrypt(personal_key, value))
            else:
                d = defer.succeed(value)
        else:
            d = defer.succeed(value)

        def got_final_value(value):
            if self.host.memory.is_session_started(profile):
                self.params[profile][(category, name)] = value
                self.host.bridge.param_update(name, value, category, profile)
                self.host.trigger.point(
                    "param_update_trigger", name, value, category, node[0], profile
                )
                return self.storage.set_ind_param(category, name, value, profile)
            else:
                raise exceptions.ProfileNotConnected

        d.addCallback(got_final_value)
        return d

    def _get_nodes_of_types(self, attr_type, node_type="@ALL@"):
        """Return all the nodes matching the given types.

        TODO: using during the dev but not anymore... remove if not needed

        @param attr_type (str): the attribute type (string, text, password, bool, int, button, list)
        @param node_type (str): keyword for filtering:
                                    @ALL@ search everywhere
                                    @GENERAL@ only search in general type
                                    @INDIVIDUAL@ only search in individual type
        @return: dict{tuple: node}: a dict {key, value} where:
            - key is a couple (attribute category, attribute name)
            - value is a node
        """
        ret = {}
        for type_node in self.dom.documentElement.childNodes:
            if (
                (node_type == "@ALL@" or node_type == "@GENERAL@")
                and type_node.nodeName == C.GENERAL
            ) or (
                (node_type == "@ALL@" or node_type == "@INDIVIDUAL@")
                and type_node.nodeName == C.INDIVIDUAL
            ):
                for cat_node in type_node.getElementsByTagName("category"):
                    cat = cat_node.getAttribute("name")
                    params = cat_node.getElementsByTagName("param")
                    for param in params:
                        if param.getAttribute("type") == attr_type:
                            ret[(cat, param.getAttribute("name"))] = param
        return ret

    def check_security_limit(self, node, security_limit):
        """Check the given node against the given security limit.
        The value NO_SECURITY_LIMIT (-1) means that everything is allowed.
        @return: True if this node can be accessed with the given security limit.
        """
        if security_limit < 0:
            return True
        if node.hasAttribute("security"):
            if int(node.getAttribute("security")) <= security_limit:
                return True
        return False

    def check_app(self, node, app):
        """Check the given node against the given app.

        @param node: parameter node
        @param app: name of the frontend requesting the parameters, or '' to get all parameters
        @return: True if this node concerns the given app.
        """
        if not app or not node.hasAttribute("app"):
            return True
        return node.getAttribute("app") == app

    def check_extra(self, node, extra):
        """Check the given node against the extra filters.

        @param node: parameter node
        @param app: name of the frontend requesting the parameters, or '' to get all parameters
        @return: True if node doesn't match category/name of extra['ignore'] list
        """
        ignore_list = extra.get('ignore')
        if not ignore_list:
            return True
        category = node.parentNode.getAttribute('name')
        name = node.getAttribute('name')
        ignore = [category, name] in ignore_list
        if ignore:
            log.debug(f"Ignoring parameter {category}/{name} as requested")
            return False
        return True


def make_options(options, selected=None):
    """Create option XML form dictionary

    @param options(dict): option's name => option's label map
    @param selected(None, str): value of selected option
        None to use first value
    @return (str): XML to use in parameters
    """
    str_list = []
    if selected is None:
        selected = next(iter(options.keys()))
    selected_found = False
    for value, label in options.items():
        if value == selected:
            selected = 'selected="true"'
            selected_found = True
        else:
            selected = ''
        str_list.append(
            f'<option value={quoteattr(value)} label={quoteattr(label)} {selected}/>'
        )
    if not selected_found:
        raise ValueError(f"selected value ({selected}) not found in options")
    return '\n'.join(str_list)