--- a/src/memory/memory.py	Sat Jan 24 00:15:01 2015 +0100
+++ b/src/memory/memory.py	Sat Jan 24 01:00:29 2015 +0100
@@ -19,11 +19,14 @@
 
 from sat.core.i18n import _
 
+from sat.core.log import getLogger
+log = getLogger(__name__)
+
 import os.path
+import copy
+from collections import namedtuple
 from ConfigParser import SafeConfigParser, NoOptionError, NoSectionError
 from uuid import uuid4
-from sat.core.log import getLogger
-log = getLogger(__name__)
 from twisted.internet import defer, reactor
 from twisted.words.protocols.jabber import jid
 from sat.core import exceptions
@@ -36,6 +39,8 @@
 from sat.tools import config as tools_config
 
 
+PresenceTuple = namedtuple("PresenceTuple", ('show', 'priority', 'statuses'))
+
 class Sessions(object):
     """Sessions are data associated to key used for a temporary moment, with optional profile checking."""
     DEFAULT_TIMEOUT = 600
@@ -212,6 +217,9 @@
         self.host = host
         self._entities_cache = {} # XXX: keep presence/last resource/other data in cache
                                   #     /!\ an entity is not necessarily in roster
+                                  #     main key is bare jid, value is a dict
+                                  #     where main key is resource, or None for bare jid
+        self._key_signals = set() # key which need a signal to frontends when updated
         self.subscriptions = {}
         self.auth_sessions = PasswordSessions()  # remember the authenticated profiles
         self.disco = Discovery(host)
@@ -228,6 +236,8 @@
         d.addCallback(lambda ignore: self.memory_data.load())
         d.chainDeferred(self.initialized)
 
+    ## Configuration ##
+
     def parseMainConf(self):
         """look for main .ini configuration file, and parse it"""
         config = SafeConfigParser(defaults=C.DEFAULT_CONFIG)
@@ -265,6 +275,24 @@
                 log.error(_("Can't load parameters from file: %s") % e)
         return False
 
+    def save_xml(self, filename):
+        """Save parameters template to xml file
+
+        @param filename (str): output file
+        @return: bool: True in case of success
+        """
+        if not filename:
+            return False
+        #TODO: need to encrypt files (at least passwords !) and set permissions
+        filename = os.path.expanduser(filename)
+        try:
+            self.params.save_xml(filename)
+            log.debug(_("Parameters saved to file: %s") % filename)
+            return True
+        except Exception as e:
+            log.error(_("Can't save parameters to file: %s") % e)
+        return False
+
     def load(self):
         """Load parameters and all memory things from db"""
         #parameters data
@@ -275,6 +303,8 @@
         @param profile: %(doc_profile)s"""
         return self.params.loadIndParams(profile)
 
+    ## Profiles/Sessions management ##
+
     def startProfileSession(self, profile):
         """"Iniatialise session for a profile
         @param profile: %(doc_profile)s"""
@@ -309,24 +339,6 @@
         except KeyError:
             log.error(_("Trying to purge roster status cache for a profile not in memory: [%s]") % profile)
 
-    def save_xml(self, filename):
-        """Save parameters template to xml file
-
-        @param filename (str): output file
-        @return: bool: True in case of success
-        """
-        if not filename:
-            return False
-        #TODO: need to encrypt files (at least passwords !) and set permissions
-        filename = os.path.expanduser(filename)
-        try:
-            self.params.save_xml(filename)
-            log.debug(_("Parameters saved to file: %s") % filename)
-            return True
-        except Exception as e:
-            log.error(_("Can't save parameters to file: %s") % e)
-        return False
-
     def getProfilesList(self):
         return self.storage.getProfilesList()
 
@@ -365,6 +377,8 @@
         self.auth_sessions.profileDelUnique(name)
         return self.params.asyncDeleteProfile(name, force)
 
+    ## History ##
+
     def addToHistory(self, from_jid, to_jid, message, type_='chat', extra=None, timestamp=None, profile=C.PROF_KEY_NONE):
         assert profile != C.PROF_KEY_NONE
         if extra is None:
@@ -393,163 +407,256 @@
             return defer.succeed([])
         return self.storage.getHistory(jid.JID(from_jid), jid.JID(to_jid), limit, between, search, profile)
 
-    def _getLastResource(self, jid_s, profile_key):
-        jid_ = jid.JID(jid_s)
-        return self.getLastResource(jid_, profile_key) or ""
-
-    def getLastResource(self, entity_jid, profile_key):
-        """Return the last resource used by an entity
-        @param entity_jid: entity jid
-        @param profile_key: %(doc_profile_key)s"""
-        data = self.getEntityData(entity_jid.userhostJID(), [C.ENTITY_LAST_RESOURCE], profile_key)
-        try:
-            return data[C.ENTITY_LAST_RESOURCE]
-        except KeyError:
-            return None
+    ## Statuses ##
 
     def _getPresenceStatuses(self, profile_key):
         ret = self.getPresenceStatuses(profile_key)
         return {entity.full():data for entity, data in ret.iteritems()}
 
     def getPresenceStatuses(self, profile_key):
-        """Get all the presence status of a profile
+        """Get all the presence statuses of a profile
+
         @param profile_key: %(doc_profile_key)s
         @return: presence data: key=entity JID, value=presence data for this entity
         """
-        profile = self.getProfileName(profile_key)
-        if not profile:
-            raise exceptions.ProfileUnknownError(_('Trying to get entity data for a non-existant profile'))
+        profile_cache = self._getProfileCache(profile_key)
         entities_presence = {}
-        for entity in self._entities_cache[profile]:
-            if "presence" in self._entities_cache[profile][entity]:
-                entities_presence[entity] = self._entities_cache[profile][entity]["presence"]
 
-        log.debug("Memory getPresenceStatus (%s)" % entities_presence)
+        for entity_jid, entity_data in profile_cache.iteritems():
+            for resource, resource_data in entity_data.iteritems():
+                full_jid = copy.copy(entity_jid)
+                full_jid.resource = resource
+                try:
+                    presence_data = self.getEntityDatum(full_jid, "presence", profile_key)
+                except KeyError:
+                    continue
+                entities_presence.setdefault(entity_jid, {})[resource or ''] = presence_data
+
         return entities_presence
 
-    def isContactConnected(self, entity_jid, profile_key):
-        """Tell from the presence information if the given contact is connected.
+    def setPresenceStatus(self, entity_jid, show, priority, statuses, profile_key):
+        """Change the presence status of an entity
+
+        @param entity_jid: jid.JID of the entity
+        @param show: show status
+        @param priority: priority
+        @param statuses: dictionary of statuses
+        @param profile_key: %(doc_profile_key)s
+        """
+        presence_data = PresenceTuple(show, priority, statuses)
+        self.updateEntityData(entity_jid, "presence", presence_data, profile_key)
+        if entity_jid.resource and show != C.PRESENCE_UNAVAILABLE:
+            # If a resource is available, bare jid should not have presence information
+            try:
+                self.delEntityDatum(entity_jid.userhostJID(), "presence", profile_key)
+            except (KeyError, exceptions.UnknownEntityError):
+                pass
+
+    ## Resources ##
+
+    def _getAllResource(self, jid_s, profile_key):
+        jid_ = jid.JID(jid_s)
+        return self.getAllResources(jid_, profile_key)
+
+    def getAllResources(self, entity_jid, profile_key):
+        """Return all resource from jid for which we have had data in this session
+
+        @param entity_jid: bare jid of the entit
+        @param profile_key: %(doc_profile_key)s
+        return (list[unicode]): list of resources
+
+        @raise exceptions.UnknownEntityError: if entity is not in cache
+        """
+        if entity_jid.resource:
+            raise ValueError("getAllResources must be used with a bare jid (got {})".format(entity_jid))
+        profile_cache = self._getProfileCache(profile_key)
+        try:
+            entity_data = profile_cache[entity_jid.userhostJID()]
+        except KeyError:
+            raise exceptions.UnknownEntityError(u"Entity {} not in cache".format(entity_jid))
+        resources= set(entity_data.keys())
+        resources.discard(None)
+        return resources
+
+    def getAvailableResources(self, entity_jid, profile_key):
+        """Return available resource for entity_jid
+
+        This method differs from getAllResources by returning only available resources
+        @param entity_jid: bare jid of the entit
+        @param profile_key: %(doc_profile_key)s
+        return (list[unicode]): list of available resources
 
-        @param entity_jid (JID): the entity to check
+        @raise exceptions.UnknownEntityError: if entity is not in cache
+        """
+        available = []
+        for resource in self.getAllResources(entity_jid, profile_key):
+            full_jid = copy.copy(entity_jid)
+            full_jid.resource = resource
+            try:
+                presence_data = self.getEntityDatum(full_jid, "presence", profile_key)
+            except KeyError:
+                log.debug("Can't get presence data for {}".format(full_jid))
+            else:
+                if presence_data.show != C.PRESENCE_UNAVAILABLE:
+                    available.append(resource)
+        return available
+
+    def _getMainResource(self, jid_s, profile_key):
+        jid_ = jid.JID(jid_s)
+        return self.getMainResource(jid_, profile_key) or ""
+
+    def getMainResource(self, entity_jid, profile_key):
+        """Return the main resource used by an entity
+
+        @param entity_jid: bare entity jid
         @param profile_key: %(doc_profile_key)s
-        @return: boolean
+        @return (unicode): main resource or None
+        """
+        if entity_jid.resource:
+            raise ValueError("getMainResource must be used with a bare jid (got {})".format(entity_jid))
+        resources = self.getAllResources(entity_jid, profile_key)
+        priority_resources = []
+        for resource in resources:
+            full_jid = copy.copy(entity_jid)
+            full_jid.resource = resource
+            try:
+                presence_data = self.getEntityDatum(full_jid, "presence", profile_key)
+            except KeyError:
+                log.debug("No presence information for {}".format(full_jid))
+                continue
+            priority_resources.append((resource, presence_data.priority))
+        try:
+            return max(priority_resources, key=lambda res_tuple: res_tuple[1])[0]
+        except ValueError:
+            log.warning("No resource found at all for {}".format(entity_jid))
+            return None
+
+    ## Entities data ##
+
+    def _getProfileCache(self, profile_key):
+        """Check profile validity and return its cache
+
+        @param profile: %(doc_profile_key)_s
+        @return (dict): profile cache
+
+        @raise exceptions.ProfileUnknownError: if profile doesn't exist
+        @raise exceptions.ProfileNotInCacheError: if there is no cache for this profile
         """
         profile = self.getProfileName(profile_key)
         if not profile:
             raise exceptions.ProfileUnknownError(_('Trying to get entity data for a non-existant profile'))
         try:
-            presence = self._entities_cache[profile][entity_jid]['presence']
-            return len([True for status in presence.values() if status[0] != 'unavailable']) > 0
+            profile_cache = self._entities_cache[profile]
         except KeyError:
-            return False
+            raise exceptions.ProfileNotInCacheError
+        return profile_cache
 
-    def setPresenceStatus(self, entity_jid, show, priority, statuses, profile_key):
-        """Change the presence status of an entity
-        @param entity_jid: jid.JID of the entity
-        @param show: show status
-        @param priotity: priotity
-        @param statuses: dictionary of statuses
-        @param profile_key: %(doc_profile_key)s
+    def setSignalOnUpdate(self, key, signal=True):
+        """Set a signal flag on the key
+
+        When the key will be updated, a signal will be sent to frontends
+        @param key: key to signal
+        @param signal(boolean): if True, do the signal
         """
-        profile = self.getProfileName(profile_key)
-        if not profile:
-            raise exceptions.ProfileUnknownError(_('Trying to get entity data for a non-existant profile'))
-        entity_data = self._getEntitiesData(entity_jid, profile)[entity_jid]
-        resource = entity_jid.resource
-        if resource:
-            try:
-                type_ = self.getEntityDatum(entity_jid.userhostJID(), 'type', profile)
-            except KeyError:
-                type_ = 'contact'
-            if type_ != 'chatroom':
-                self.updateEntityData(entity_jid.userhostJID(), C.ENTITY_LAST_RESOURCE, resource, profile)
-        entity_data.setdefault("presence", {})[resource or ''] = (show, priority, statuses)
+        if signal:
+            self._key_signals.add(key)
+        else:
+            self._key_signals.discard(key)
+
+    def getAllEntitiesIter(self, with_bare=False, profile_key=C.PROF_KEY_NONE):
+        """Return an iterator of full jids of all entities in cache
 
-    def _getEntitiesData(self, entity_jid, profile):
-        """Get data dictionary for entities
-        @param entity_jid: JID of the entity, or C.ENTITY_ALL for all entities)
-        @param profile: %(doc_profile)s
-        @return: entities_data (key=jid, value=entity_data)
-        @raise: exceptions.ProfileNotInCacheError if profile is not in cache
+        @param with_bare: if True, include bare jids
+        @param profile_key: %(doc_profile_key)s
+        @return (list[unicode]): list of jids
         """
-        if not profile in self._entities_cache:
-            raise exceptions.ProfileNotInCacheError
-        if entity_jid == C.ENTITY_ALL:
-            entities_data = self._entities_cache[profile]
-        else:
-            entity_data = self._entities_cache[profile].setdefault(entity_jid, {})
-            entities_data = {entity_jid: entity_data}
-        return entities_data
-
-    def _updateEntityResources(self, entity_jid, profile):
-        """Add a known resource to bare entity_jid cache
-        @param entity_jid: full entity_jid (with resource)
-        @param profile: %(doc_profile)s
-        """
-        assert(entity_jid.resource)
-        entity_data = self._getEntitiesData(entity_jid.userhostJID(), profile)[entity_jid.userhostJID()]
-        resources = entity_data.setdefault('resources', set())
-        resources.add(entity_jid.resource)
+        profile_cache = self._getProfileCache(profile_key)
+        # we construct a list of all known full jids (bare jid of entities x resources)
+        for bare_jid, entity_data in profile_cache.iteritems():
+            for resource in entity_data.iterkeys():
+                if resource is None:
+                    continue
+                full_jid = copy.copy(bare_jid)
+                full_jid.resource = resource
+                yield full_jid
 
     def updateEntityData(self, entity_jid, key, value, profile_key):
         """Set a misc data for an entity
-        @param entity_jid: JID of the entity, or C.ENTITY_ALL to update all entities)
+
+        If key was registered with setSignalOnUpdate, a signal will be sent to frontends
+        @param entity_jid: JID of the entity, C.ENTITY_ALL_RESOURCES for all resources of all entities,
+                           C.ENTITY_ALL for all entities (all resources + bare jids)
         @param key: key to set (eg: "type")
         @param value: value for this key (eg: "chatroom")
         @param profile_key: %(doc_profile_key)s
         """
-        profile = self.getProfileName(profile_key)
-        if not profile:
-            raise exceptions.ProfileUnknownError(_('Trying to get entity data for a non-existant profile'))
-        entities_data = self._getEntitiesData(entity_jid, profile)
-        if entity_jid != C.ENTITY_ALL and entity_jid.resource:
-            self._updateEntityResources(entity_jid, profile)
+        profile_cache = self._getProfileCache(profile_key)
+        if entity_jid in (C.ENTITY_ALL_RESOURCES, C.ENTITY_ALL):
+            entities = self.getAllEntitiesIter(entity_jid==C.ENTITY_ALL, profile_key)
+        else:
+            entities = (entity_jid,)
+
+        for jid_ in entities:
+            entity_data = profile_cache.setdefault(jid_.userhostJID(),{}).setdefault(jid_.resource, {})
 
-        for jid_ in entities_data:
-            entity_data = entities_data[jid_]
-            if value == C.PROF_KEY_NONE and key in entity_data:
-                del entity_data[key]
-            else:
-                entity_data[key] = value
-            if isinstance(value, basestring):
-                self.host.bridge.entityDataUpdated(jid_.full(), key, value, profile)
+            entity_data[key] = value
+            if key in self._key_signals:
+                if not isinstance(value, basestring):
+                    log.error(u"Setting a non string value ({}) for a key ({}) which has a signal flag".format(value, key))
+                else:
+                    self.host.bridge.entityDataUpdated(jid_.full(), key, value, self.getProfileName(profile_key))
 
-    def delEntityData(self, entity_jid, key, profile_key):
-        """Delete data for an entity
-        @param entity_jid: JID of the entity, or C.ENTITY_ALL to delete data from all entities)
+    def delEntityDatum(self, entity_jid, key, profile_key):
+        """Delete a data for an entity
+
+        @param entity_jid: JID of the entity, C.ENTITY_ALL_RESOURCES for all resources of all entities,
+                           C.ENTITY_ALL for all entities (all resources + bare jids)
         @param key: key to delete (eg: "type")
         @param profile_key: %(doc_profile_key)s
+
+        @raise exceptions.UnknownEntityError: if entity is not in cache
+        @raise KeyError: key is not in cache
         """
-        entities_data = self._getEntitiesData(entity_jid, profile_key)
-        for entity_jid in entities_data:
-            entity_data = entities_data[entity_jid]
+        profile_cache = self._getProfileCache(profile_key)
+        if entity_jid in (C.ENTITY_ALL_RESOURCES, C.ENTITY_ALL):
+            entities = self.getAllEntitiesIter(entity_jid==C.ENTITY_ALL, profile_key)
+        else:
+            entities = (entity_jid,)
+
+        for jid_ in entities:
+            try:
+                entity_data = profile_cache[jid_.userhostJID()][jid_.resource]
+            except KeyError:
+                raise exceptions.UnknownEntityError(u"Entity {} not in cache".format(jid_))
             try:
                 del entity_data[key]
-            except KeyError:
-                log.debug("Key [%s] doesn't exist for [%s] in entities_cache" % (key, entity_jid.full()))
+            except KeyError as e:
+                if entity_jid in (C.ENTITY_ALL_RESOURCES, C.ENTITY_ALL):
+                    continue # we ignore KeyError when deleting keys from several entities
+                else:
+                    raise e
 
-    def getEntityData(self, entity_jid, keys_list, profile_key):
+    def getEntityData(self, entity_jid, keys_list=None, profile_key=C.PROF_KEY_NONE):
         """Get a list of cached values for entity
 
         @param entity_jid: JID of the entity
-        @param keys_list (iterable): list of keys to get, empty list for everything
+        @param keys_list (iterable,None): list of keys to get, None for everything
         @param profile_key: %(doc_profile_key)s
         @return: dict withs values for each key in keys_list.
                  if there is no value of a given key, resulting dict will
                  have nothing with that key nether
-        @raise: exceptions.UnknownEntityError if entity is not in cache
+
+        @raise exceptions.UnknownEntityError: if entity is not in cache
         """
-        profile = self.getProfileName(profile_key)
-        if not profile:
-            raise exceptions.ProfileUnknownError(_('Trying to get entity data for a non-existant profile'))
-        entity_data = self._getEntitiesData(entity_jid, profile)[entity_jid]
-        if not keys_list:
+        profile_cache = self._getProfileCache(profile_key)
+        try:
+            entity_data = profile_cache[entity_jid.userhostJID()][entity_jid.resource]
+        except KeyError:
+            raise exceptions.UnknownEntityError(u"Entity {} not in cache (was requesting {})".format(entity_jid, keys_list))
+        if keys_list is None:
             return entity_data
-        ret = {}
-        for key in keys_list:
-            if key in entity_data:
-                ret[key] = entity_data[key]
-        return ret
+
+        return {key: entity_data[key] for key in keys_list if key in entity_data}
 
     def getEntityDatum(self, entity_jid, key, profile_key):
         """Get a datum from entity
@@ -559,34 +666,36 @@
         @param profile_key: %(doc_profile_key)s
         @return: requested value
 
-        @raise: exceptions.UnknownEntityError if entity is not in cache
-        @raise: KeyError if there is no value for this key and this entity
+        @raise exceptions.UnknownEntityError: if entity is not in cache
+        @raise KeyError: if there is no value for this key and this entity
         """
         return self.getEntityData(entity_jid, (key,), profile_key)[key]
 
     def delEntityCache(self, entity_jid, delete_all_resources=True, profile_key=C.PROF_KEY_NONE):
-        """Remove cached data for entity
+        """Remove all cached data for entity
+
         @param entity_jid: JID of the entity to delete
-        @param delete_all_resources: if True also delete all known resources form cache
+        @param delete_all_resources: if True also delete all known resources from cache (a bare jid must be given in this case)
         @param profile_key: %(doc_profile_key)s
+
+        @raise exceptions.UnknownEntityError: if entity is not in cache
         """
-        profile = self.getProfileName(profile_key)
-        if not profile:
-            raise exceptions.ProfileUnknownError(_('Trying to get entity data for a non-existant profile'))
-        to_delete = set([entity_jid])
+        profile_cache = self._getProfileCache(profile_key)
 
         if delete_all_resources:
             if entity_jid.resource:
                 raise ValueError(_("Need a bare jid to delete all resources"))
-            entity_data = self._getEntitiesData(entity_jid, profile)[entity_jid]
-            resources = entity_data.setdefault('resources', set())
-            to_delete.update([jid.JID("%s/%s" % (entity_jid.userhost(), resource)) for resource in resources])
+            try:
+                del profile_cache[entity_jid]
+            except KeyError:
+                raise exceptions.UnknownEntityError(u"Entity {} not in cache".format(entity_jid))
+        else:
+            try:
+                del profile_cache[entity_jid.userhostJID()][entity_jid.resource]
+            except KeyError:
+                raise exceptions.UnknownEntityError(u"Entity {} not in cache".format(entity_jid))
 
-        for entity in to_delete:
-            try:
-                del self._entities_cache[profile][entity]
-            except KeyError:
-                log.warning(_("Can't delete entity [{}]: not in cache").format(entity.full()))
+    ## Encryption ##
 
     def encryptValue(self, value, profile):
         """Encrypt a value for the given profile. The personal key must be loaded
@@ -644,6 +753,8 @@
         d = PersistentDict(C.MEMORY_CRYPTO_NAMESPACE, profile).load()
         return d.addCallback(gotIndMemory).addCallback(done)
 
+    ## Subscription requests ##
+
     def addWaitingSub(self, type_, entity_jid, profile_key):
         """Called when a subcription request is received"""
         profile = self.getProfileName(profile_key)
@@ -670,6 +781,8 @@
 
         return self.subscriptions[profile]
 
+    ## Parameters ##
+
     def getStringParamA(self, name, category, attr="value", profile_key=C.PROF_KEY_NONE):
         return self.params.getStringParamA(name, category, attr, profile_key)
 
@@ -705,3 +818,21 @@
 
     def setDefault(self, name, category, callback, errback=None):
         return self.params.setDefault(name, category, callback, errback)
+
+    ## Misc ##
+
+    def isEntityAvailable(self, entity_jid, profile_key):
+        """Tell from the presence information if the given entity is available.
+
+        @param entity_jid (JID): the entity to check (if bare jid is used, all resources are tested)
+        @param profile_key: %(doc_profile_key)s
+        @return (bool): True if entity is available
+        """
+        if not entity_jid.resource:
+            return bool(self.getAvailableResources) # is any resource is available, entity is available
+        try:
+            presence_data = self.getEntityDatum(entity_jid, "presence", profile_key)
+        except KeyError:
+            log.debug("No presence information for {}".format(entity_jid))
+            return False
+        return presence_data.show != C.PRESENCE_UNAVAILABLE