--- a/sat/plugins/plugin_misc_identity.py	Tue Apr 14 20:36:24 2020 +0200
+++ b/sat/plugins/plugin_misc_identity.py	Tue Apr 14 21:00:33 2020 +0200
@@ -1,9 +1,6 @@
 #!/usr/bin/env python3
 
-
-# SAT plugin for managing xep-0054
 # Copyright (C) 2009-2020 Jérôme Poisson (goffi@goffi.org)
-# Copyright (C) 2014 Emmanuel Gil Peyrot (linkmauve@linkmauve.fr)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,15 +15,21 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from collections import namedtuple
+from pathlib import Path
+from twisted.internet import defer
+from twisted.words.protocols.jabber import jid
 from sat.core.i18n import _
 from sat.core.constants import Const as C
 from sat.core import exceptions
 from sat.core.log import getLogger
+from sat.memory import persistent
+from sat.tools import image
+from sat.tools import utils
+from sat.tools.common import data_format
+
 
 log = getLogger(__name__)
-from twisted.internet import defer
-from twisted.words.protocols.jabber import jid
-import os.path
 
 
 PLUGIN_INFO = {
@@ -34,95 +37,529 @@
     C.PI_IMPORT_NAME: "IDENTITY",
     C.PI_TYPE: C.PLUG_TYPE_MISC,
     C.PI_PROTOCOLS: [],
-    C.PI_DEPENDENCIES: ["XEP-0054"],
-    C.PI_RECOMMENDATIONS: [],
+    C.PI_DEPENDENCIES: [],
+    C.PI_RECOMMENDATIONS: ["XEP-0045"],
     C.PI_MAIN: "Identity",
     C.PI_HANDLER: "no",
     C.PI_DESCRIPTION: _("""Identity manager"""),
 }
 
+Callback = namedtuple("Callback", ("get", "set", "priority"))
 
-class Identity(object):
+
+class Identity:
+
     def __init__(self, host):
         log.info(_("Plugin Identity initialization"))
         self.host = host
-        self._v = host.plugins["XEP-0054"]
+        self._m = host.plugins.get("XEP-0045")
+        self.metadata = {
+            "avatar": {
+                "type": dict,
+                # convert avatar path to avatar metadata (and check validity)
+                "set_data_filter": self.avatarSetDataFilter,
+                # update profile avatar, so all frontends are aware
+                "set_post_treatment": self.avatarSetPostTreatment,
+                "update_is_new_data": self.avatarUpdateIsNewData,
+                "update_data_filter": self.avatarUpdateDataFilter,
+                # we store the metadata in database, to restore it on next connection
+                # (it is stored only for roster entities)
+                "store": True,
+            },
+            "nicknames": {
+                "type": list,
+                # accumulate all nicknames from all callbacks in a list instead
+                # of returning only the data from the first successful callback
+                "get_all": True,
+                # append nicknames from roster, resource, etc.
+                "get_post_treatment": self.nicknamesGetPostTreatment,
+                "update_is_new_data": self.nicknamesUpdateIsNewData,
+                "store": True,
+            },
+        }
+        host.trigger.add("roster_update", self._rosterUpdateTrigger)
+        host.memory.setSignalOnUpdate("avatar")
+        host.memory.setSignalOnUpdate("nicknames")
         host.bridge.addMethod(
             "identityGet",
             ".plugin",
-            in_sign="ss",
-            out_sign="a{ss}",
+            in_sign="sasbs",
+            out_sign="s",
             method=self._getIdentity,
             async_=True,
         )
         host.bridge.addMethod(
             "identitySet",
             ".plugin",
-            in_sign="a{ss}s",
+            in_sign="ss",
             out_sign="",
             method=self._setIdentity,
             async_=True,
         )
+        host.bridge.addMethod(
+            "avatarGet",
+            ".plugin",
+            in_sign="sbs",
+            out_sign="s",
+            method=self._getAvatar,
+            async_=True,
+        )
+        host.bridge.addMethod(
+            "avatarSet",
+            ".plugin",
+            in_sign="sss",
+            out_sign="",
+            method=self._setAvatar,
+            async_=True,
+        )
 
-    def _getIdentity(self, jid_str, profile):
-        jid_ = jid.JID(jid_str)
+    async def profileConnecting(self, client):
+        # we restore known identities from database
+        client._identity_storage = persistent.LazyPersistentBinaryDict(
+            "identity", client.profile)
+
+        stored_data = await client._identity_storage.all()
+
+        self.host.memory.storage.getPrivates(
+            namespace="identity", binary=True, profile=client.profile)
+
+        to_delete = []
+
+        for key, value in stored_data.items():
+            entity_s, name = key.split('\n')
+            if name not in self.metadata.keys():
+                log.debug(f"removing {key} from storage: not an allowed metadata name")
+                to_delete.append(key)
+                continue
+            entity = jid.JID(entity_s)
+
+            if name == 'avatar':
+                if value is not None:
+                    try:
+                        cache_uid = value['cache_uid']
+                        if not cache_uid:
+                            raise ValueError
+                    except (ValueError, KeyError):
+                        log.warning(
+                            f"invalid data for {entity} avatar, it will be deleted: "
+                            f"{value}")
+                        to_delete.append(key)
+                        continue
+                    cache = self.host.common_cache.getMetadata(cache_uid)
+                    if cache is None:
+                        log.debug(
+                            f"purging avatar for {entity}: it is not in cache anymore")
+                        to_delete.append(key)
+                        continue
+
+            self.host.memory.updateEntityData(
+                client, entity, name, value, silent=True
+            )
+
+        for key in to_delete:
+            await client._identity_storage.adel(key)
+
+    def _rosterUpdateTrigger(self, client, roster_item):
+        old_item = client.roster.getItem(roster_item.jid)
+        if old_item is None or old_item.name != roster_item.name:
+            log.debug(
+                f"roster nickname has been updated to {roster_item.name!r} for "
+                f"{roster_item.jid}"
+            )
+            defer.ensureDeferred(
+                self.update(
+                    client,
+                    "nicknames",
+                    [roster_item.name],
+                    roster_item.jid
+                )
+            )
+        return True
+
+    def register(self, metadata_name, cb_get, cb_set, priority=0):
+        """Register callbacks to handle identity metadata
+
+        @param metadata_name(str): name of metadata can be:
+            - avatar
+            - nicknames
+        @param cb_get(coroutine, Deferred): method to retrieve a metadata
+            the method will get client and metadata names to retrieve as arguments.
+        @param cb_set(coroutine, Deferred): method to set a metadata
+            the method will get client, metadata name to set, and value as argument.
+        @param priority(int): priority of this method for the given metadata.
+            methods with bigger priorities will be called first
+        """
+        if not metadata_name in self.metadata.keys():
+            raise ValueError(f"Invalid metadata_name: {metadata_name!r}")
+        callback = Callback(get=cb_get, set=cb_set, priority=priority)
+        cb_list = self.metadata[metadata_name].setdefault('callbacks', [])
+        cb_list.append(callback)
+        cb_list.sort(key=lambda c: c.priority, reverse=True)
+
+    def getIdentityJid(self, client, peer_jid):
+        """Return jid to use to set identity metadata
+
+        if it's a jid of a room occupant, full jid will be used
+        otherwise bare jid will be used
+        if None, bare jid of profile will be used
+        @return (jid.JID): jid to use for avatar
+        """
+        if peer_jid is None:
+            return client.jid.userhostJID()
+        if self._m is None:
+            return peer_jid.userhostJID()
+        else:
+            return self._m.getBareOrFull(client, peer_jid)
+
+    def checkType(self, metadata_name, value):
+        """Check that type used for a metadata is the one declared in self.metadata"""
+        value_type = self.metadata[metadata_name]["type"]
+        if not isinstance(value, value_type):
+            raise ValueError(
+                f"{value} has wrong type: it is {type(value)} while {value_type} was "
+                f"expected")
+
+    async def get(self, client, metadata_name, entity, use_cache=True):
+        """Retrieve identity metadata of an entity
+
+        if metadata is already in cache, it is returned. Otherwise, registered callbacks
+        will be tried in priority order (bigger to lower)
+        @param metadata_name(str): name of the metadata
+            must be one of self.metadata key
+            the name will also be used as entity data name in host.memory
+        @param entity(jid.JID, None): entity for which avatar is requested
+            None to use profile's jid
+        @param use_cache(bool): if False, cache won't be checked
+        """
+        entity = self.getIdentityJid(client, entity)
+        try:
+            metadata = self.metadata[metadata_name]
+        except KeyError:
+            raise ValueError(f"Invalid metadata name: {metadata_name!r}")
+        get_all = metadata.get('get_all', False)
+        if use_cache:
+            try:
+                data = self.host.memory.getEntityDatum(
+                    client, entity, metadata_name)
+            except (KeyError, exceptions.UnknownEntityError):
+                pass
+            else:
+                return data
+
+        try:
+            callbacks = metadata['callbacks']
+        except KeyError:
+            log.warning(_("No callback registered for {metadata_name}")
+                        .format(metadata_name=metadata_name))
+            return [] if get_all else None
+
+        if get_all:
+            all_data = []
+
+        for callback in callbacks:
+            try:
+                data = await defer.ensureDeferred(callback.get(client, entity))
+            except exceptions.CancelError:
+                continue
+            except Exception as e:
+                log.warning(
+                    _("Error while trying to get {metadata_name} with {callback}: {e}")
+                    .format(callback=callback.get, metadata_name=metadata_name, e=e))
+            else:
+                if data:
+                    self.checkType(metadata_name, data)
+                    if get_all:
+                        all_data.extend(data)
+                    else:
+                        break
+        else:
+            data = None
+
+        if get_all:
+            data = all_data
+
+        post_treatment = metadata.get("get_post_treatment")
+        if post_treatment is not None:
+            data = await utils.asDeferred(post_treatment, client, entity, data)
+
+        self.host.memory.updateEntityData(
+            client, entity, metadata_name, data)
+
+        if metadata.get('store', False):
+            key = f"{entity}\n{metadata_name}"
+            await client._identity_storage.aset(key, data)
+
+        return data
+
+    async def set(self, client, metadata_name, data, entity=None):
+        """Set identity metadata for an entity
+
+        Registered callbacks will be tried in priority order (bigger to lower)
+        @param metadata_name(str): name of the metadata
+            must be one of self.metadata key
+            the name will also be used to set entity data in host.memory
+        @param data(object): value to set
+        @param entity(jid.JID, None): entity for which avatar is requested
+            None to use profile's jid
+        """
+        entity = self.getIdentityJid(client, entity)
+        metadata = self.metadata[metadata_name]
+        data_filter = metadata.get("set_data_filter")
+        if data_filter is not None:
+            data = await utils.asDeferred(data_filter, client, entity, data)
+        self.checkType(metadata_name, data)
+
+        try:
+            callbacks = metadata['callbacks']
+        except KeyError:
+            log.warning(_("No callback registered for {metadata_name}")
+                        .format(metadata_name=metadata_name))
+            return exceptions.FeatureNotFound(f"Can't set {metadata_name} for {entity}")
+
+        for callback in callbacks:
+            try:
+                await defer.ensureDeferred(callback.set(client, data, entity))
+            except exceptions.CancelError:
+                continue
+            except Exception as e:
+                log.warning(
+                    _("Error while trying to set {metadata_name} with {callback}: {e}")
+                    .format(callback=callback.set, metadata_name=metadata_name, e=e))
+            else:
+                break
+        else:
+            raise exceptions.FeatureNotFound(f"Can't set {metadata_name} for {entity}")
+
+        post_treatment = metadata.get("set_post_treatment")
+        if post_treatment is not None:
+            await utils.asDeferred(post_treatment, client, entity, data)
+
+    async def update(self, client, metadata_name, data, entity):
+        """Update a metadata in cache
+
+        This method may be called by plugins when an identity metadata is available.
+        """
+        entity = self.getIdentityJid(client, entity)
+        metadata = self.metadata[metadata_name]
+
+        try:
+            cached_data = self.host.memory.getEntityDatum(
+                client, entity, metadata_name)
+        except (KeyError, exceptions.UnknownEntityError):
+            # metadata is not cached, we do the update
+            pass
+        else:
+            # metadata is cached, we check if the new value differs from the cached one
+            try:
+                update_is_new_data = metadata["update_is_new_data"]
+            except KeyError:
+                update_is_new_data = self.defaultUpdateIsNewData
+
+            if not update_is_new_data(client, entity, cached_data, data):
+                if cached_data is None:
+                    log.debug(
+                        f"{metadata_name} for {entity} is already disabled, nothing to "
+                        f"do")
+                else:
+                    log.debug(
+                        f"{metadata_name} for {entity} is already in cache, nothing to "
+                        f"do")
+                return
+
+        # we can't use the cache, so we do the update
+
+        log.debug(f"updating {metadata_name} for {entity}")
+
+        if metadata.get('get_all', False):
+            # get_all is set, meaning that we have to check all plugins
+            # so we first delete current cache
+            self.host.memory.delEntityDatum(client, entity, metadata_name)
+            # then fill it again by calling get, which will retrieve all values
+            await self.get(client, metadata_name, entity)
+            return
+
+        if data is not None:
+            data_filter = metadata['update_data_filter']
+            if data_filter is not None:
+                data = await utils.asDeferred(data_filter, client, entity, data)
+            self.checkType(metadata_name, data)
+
+        self.host.memory.updateEntityData(client, entity, metadata_name, data)
+
+        if metadata.get('store', False):
+            key = f"{entity}\n{metadata_name}"
+            await client._identity_storage.aset(key, data)
+
+    def defaultUpdateIsNewData(self, client, entity, cached_data, new_data):
+        return new_data != cached_data
+
+    def _getAvatar(self, entity, use_cache, profile):
         client = self.host.getClient(profile)
-        return self.getIdentity(client, jid_)
+        entity = jid.JID(entity) if entity else None
+        d = defer.ensureDeferred(self.get(client, "avatar", entity, use_cache))
+        d.addCallback(lambda data: data_format.serialise(data))
+        return d
+
+    def _setAvatar(self, file_path, entity, profile_key=C.PROF_KEY_NONE):
+        client = self.host.getClient(profile_key)
+        entity = jid.JID(entity) if entity else None
+        return defer.ensureDeferred(
+            self.set(client, "avatar", file_path, entity))
+
+    async def avatarSetDataFilter(self, client, entity, file_path):
+        """Convert avatar file path to dict data"""
+        file_path = Path(file_path)
+        if not file_path.is_file():
+            raise ValueError(f"There is no file at {file_path} to use as avatar")
+        avatar_data = {
+            'path': file_path,
+            'media_type': image.guess_type(file_path),
+        }
+        media_type = avatar_data['media_type']
+        if media_type is None:
+            raise ValueError(f"Can't identify type of image at {file_path}")
+        if not media_type.startswith('image/'):
+            raise ValueError(f"File at {file_path} doesn't appear to be an image")
+        return avatar_data
+
+    async def avatarSetPostTreatment(self, client, entity, avatar_data):
+        """Update our own avatar"""
+        await self.update(client, "avatar", avatar_data, entity)
+
+    def avatarBuildMetadata(self, path, media_type=None, cache_uid=None):
+        """Helper method to generate avatar metadata
+
+        @param path(str, Path, None): path to avatar file
+            avatar file must be in cache
+            None if avatar is explicitely not set
+        @param media_type(str, None): type of the avatar file (MIME type)
+        @param cache_uid(str, None): UID of avatar in cache
+        @return (dict, None): avatar metadata
+            None if avatar is not set
+        """
+        if path is None:
+            return None
+        else:
+            if cache_uid is None:
+                raise ValueError("cache_uid must be set if path is set")
+            path = Path(path)
+            if media_type is None:
+                media_type = image.guess_type(path)
+
+            return {
+                "path": path,
+                "media_type": media_type,
+                "cache_uid": cache_uid,
+            }
+
+    def avatarUpdateIsNewData(self, client, entity, cached_data, file_path):
+        if cached_data is None:
+            return file_path is not None
 
-    @defer.inlineCallbacks
-    def getIdentity(self, client, jid_):
+        if file_path is not None and file_path == cached_data['path']:
+            if file_path is None:
+                log.debug(
+                    f"Avatar is already disabled for {entity}, nothing to do")
+            else:
+                log.debug(
+                    f"Avatar at {file_path} is already used by {entity}, nothing "
+                    f"to do")
+            return
+
+    async def avatarUpdateDataFilter(self, client, entity, data):
+        if not isinstance(data, dict):
+            raise ValueError(f"Invalid data type ({type(data)}), a dict is expected")
+        mandatory_keys = {'path', 'cache_uid'}
+        if not data.keys() >= mandatory_keys:
+            raise ValueError(f"missing avatar data keys: {mandatory_keys - data.keys()}")
+        return data
+
+    async def nicknamesGetPostTreatment(self, client, entity, plugin_nicknames):
+        """Prepend nicknames from core locations + set default nickname
+
+        nicknames are checked from many locations, there is always at least
+        one nickname. First nickname of the list can be used in priority.
+        Nicknames are appended in this order:
+            - roster, plugins set nicknames
+            - if no nickname is found, user part of jid is then used, or bare jid
+              if there is no user part.
+        For MUC, room nick is always put first
+        """
+        # we first check roster
+        nicknames = []
+        if entity.resource:
+            # getIdentityJid let the resource only if the entity is a MUC room
+            # occupant jid
+            nicknames.append(entity.resource)
+
+        roster_item = client.roster.getItem(entity.userhostJID())
+        if roster_item is not None and roster_item.name:
+            # user set name has priority over entity set name
+            nicknames.append(roster_item.name)
+
+        nicknames.extend(plugin_nicknames)
+
+        if not nicknames:
+            if entity.user:
+                nicknames.append(entity.user.capitalize())
+            else:
+                nicknames.append(entity.userhost())
+
+        # we remove duplicates while preserving order with dict
+        return list(dict.fromkeys(nicknames))
+
+    def nicknamesUpdateIsNewData(self, client, entity, cached_data, new_nicknames):
+        return not set(new_nicknames).issubset(cached_data)
+
+    def _getIdentity(self, entity_s, metadata_filter, use_cache, profile):
+        entity = jid.JID(entity_s)
+        client = self.host.getClient(profile)
+        d = defer.ensureDeferred(
+            self.getIdentity(client, entity, metadata_filter, use_cache))
+        d.addCallback(lambda data: data_format.serialise(data))
+        return d
+
+    async def getIdentity(
+        self, client, entity=None, metadata_filter=None, use_cache=True):
         """Retrieve identity of an entity
 
-        @param jid_(jid.JID): entity to check
-        @return (dict(unicode, unicode)): identity data where key can be:
-            - nick: nickname of the entity
-                nickname is checked from, in this order:
-                    roster, vCard, user part of jid
-            cache is used when possible
+        @param entity(jid.JID, None): entity to check
+        @param metadata_filter(list[str], None): if not None or empty, only return
+            metadata in this filter
+        @param use_cache(bool): if False, cache won't be checked
+            should be True most of time, to avoid useless network requests
+        @return (dict): identity data
         """
         id_data = {}
-        # we first check roster
-        roster_item = yield client.roster.getItem(jid_.userhostJID())
-        if roster_item is not None and roster_item.name:
-            id_data["nick"] = roster_item.name
-        elif jid_.resource and self._v.isRoom(client, jid_):
-            id_data["nick"] = jid_.resource
+
+        if not metadata_filter:
+            metadata_names = self.metadata.keys()
         else:
-            #  and finally then vcard
-            nick = yield self._v.getNick(client, jid_)
-            if nick:
-                id_data["nick"] = nick
-            elif jid_.user:
-                id_data["nick"] = jid_.user.capitalize()
-            else:
-                id_data["nick"] = jid_.userhost()
+            metadata_names = metadata_filter
 
-        try:
-            avatar_path = id_data["avatar"] = yield self._v.getAvatar(
-                client, jid_, cache_only=False
-            )
-        except exceptions.NotFound:
-            pass
-        else:
-            if avatar_path:
-                id_data["avatar_basename"] = os.path.basename(avatar_path)
-            else:
-                del id_data["avatar"]
+        for metadata_name in metadata_names:
+            id_data[metadata_name] = await self.get(
+                client, metadata_name, entity, use_cache)
+
+        return id_data
 
-        defer.returnValue(id_data)
-
-    def _setIdentity(self, id_data, profile):
+    def _setIdentity(self, id_data_s, profile):
         client = self.host.getClient(profile)
-        return self.setIdentity(client, id_data)
+        id_data = data_format.deserialise(id_data_s)
+        return defer.ensureDeferred(self.setIdentity(client, id_data))
 
-    def setIdentity(self, client, id_data):
+    async def setIdentity(self, client, id_data):
         """Update profile's identity
 
-        @param id_data(dict[unicode, unicode]): data to update, key can be:
-            - nick: nickname
-                the vCard will be updated
+        @param id_data(dict): data to update, key can be on of self.metadata keys
         """
-        if list(id_data.keys()) != ["nick"]:
-            raise NotImplementedError("Only nick can be updated for now")
-        if "nick" in id_data:
-            return self._v.setNick(client, id_data["nick"])
+        if not id_data.keys() <= self.metadata.keys():
+            raise ValueError(
+                f"Invalid metadata names: {id_data.keys() - self.metadata.keys()}")
+        for metadata_name, data in id_data.items():
+            try:
+                await self.set(client, metadata_name, data)
+            except Exception as e:
+                log.warning(
+                    _("Can't set metadata {metadata_name!r}: {reason}")
+                    .format(metadata_name=metadata_name, reason=e))