diff src/plugins/plugin_misc_groupblog.py @ 1217:318eab3f93f8

plugin XEP-0060, groupblog: avoid unecessary pubsub errors while doing massive requests: - don't try to retrieve items from non accessible nodes - don't try to subscribe to non accessible or already subscribed nodes
author souliane <souliane@mailoo.org>
date Mon, 22 Sep 2014 20:49:13 +0200
parents 4be53c14845e
children 0b87d029f0a3
line wrap: on
line diff
--- a/src/plugins/plugin_misc_groupblog.py	Mon Sep 22 20:34:29 2014 +0200
+++ b/src/plugins/plugin_misc_groupblog.py	Mon Sep 22 20:49:13 2014 +0200
@@ -481,6 +481,7 @@
     @defer.inlineCallbacks
     def _itemsConstruction(self, items, pub_jid, client):
         """ Transforms items to group blog data and manage comments node
+
         @param items: iterable of items
         @param pub_jid: jid of the publisher or None to use items data
         @param client: SatXMPPClient instance
@@ -494,15 +495,20 @@
             except AttributeError:
                 pass
             ret.append(gbdata)
-            # if there is a comments node, we subscribe to it
-            if "comments_node" in gbdata:
+            # every comments node must be subscribed, except if we are the publisher (we are already subscribed in this case)
+            if "comments_node" in gbdata and pub_jid.userhostJID() != client.jid.userhostJID():
                 try:
-                    # every comments node must be subscribed, except if we are the publisher (we are already subscribed in this case)
-                    if pub_jid.userhostJID() != client.jid.userhostJID():
-                        self.host.plugins["XEP-0060"].subscribe(jid.JID(gbdata["comments_service"]), gbdata["comments_node"],
-                                                                profile_key=client.profile)
+                    service = jid.JID(gbdata["comments_service"])
+                    node = gbdata["comments_node"]
                 except KeyError:
                     log.warning("Missing key for comments")
+                    continue
+                # TODO: see if it is really needed to check for not subscribing twice to the node
+                # It previously worked without this check, but the pubsub service logs were polluted
+                # or, if in debug mode, it made sat-pubsub very difficult to debug.
+                subscribed_nodes = yield self.host.plugins['XEP-0060'].listSubscribedNodes(service, profile=client.profile)
+                if node not in subscribed_nodes:  # avoid sat-pubsub "SubscriptionExists" error
+                    self.host.plugins["XEP-0060"].subscribe(service, node, profile_key=client.profile)
         defer.returnValue(ret)
 
     def __getGroupBlogs(self, pub_jid_s, max_items=10, item_ids=None, profile_key=C.PROF_KEY_NONE):
@@ -645,6 +651,7 @@
             publishers_jids = publishers
         return self.getMassiveLastGroupBlogs(publishers_type, publishers_jids, max_items, profile_key)
 
+    @defer.inlineCallbacks
     def getMassiveLastGroupBlogs(self, publishers_type, publishers, max_items=10, profile_key=C.PROF_KEY_NONE):
         """Get the last published microblogs for a list of groups or jids
         @param publishers_type: type of the list of publishers (one of "GROUP" or "JID" or "ALL")
@@ -652,58 +659,36 @@
         @param max_items: how many microblogs we want to get
         @param profile_key: profile key
         """
-
-        def sendResult(result):
-            """send result of DeferredList (dict of jid => microblogs) to the calling method"""
-
-            ret = {}
-
-            for (success, value) in result:
-                if success:
-                    source_jid, data = value
-                    ret[source_jid] = data
-
-            return ret
-
-        def initialised(result):
-            profile, client = result
-
-            if publishers_type == "ALL":
-                contacts = client.roster.getItems()
-                jids = [contact.jid.userhostJID() for contact in contacts]
-            elif publishers_type == "GROUP":
-                jids = []
-                for _group in publishers:
-                    jids.extend(client.roster.getJidsFromGroup(_group))
-            elif publishers_type == 'JID':
-                jids = publishers
-            else:
-                raise UnknownType
-
-            mblogs = []
-
-            for jid_ in jids:
-                d = self.host.plugins["XEP-0060"].getItems(client.item_access_pubsub, self.getNodeName(jid_),
-                                                           max_items=max_items, profile_key=profile_key)
-                d.addCallback(self._itemsConstruction, jid_, client)
-                d.addCallback(lambda gbdata, source_jid: (source_jid, gbdata), jid_.full())
-
-                mblogs.append(d)
-            # consume the failure "StanzaError with condition u'item-not-found'"
-            # when the node doesn't exist (e.g that JID hasn't posted any message)
-            dlist = defer.DeferredList(mblogs, consumeErrors=True)
-            dlist.addCallback(sendResult)
-
-            return dlist
-
         #TODO: custom exception
         if publishers_type not in ["GROUP", "JID", "ALL"]:
             raise Exception("Bad call, unknown publishers_type")
         if publishers_type == "ALL" and publishers:
             raise Exception("Publishers list must be empty when getting microblogs for all contacts")
-        return self._initialise(profile_key).addCallback(initialised)
+        profile, client = yield self._initialise(profile_key)
         #TODO: we need to use the server corresponding the the host of the jid
 
+        if publishers_type == "ALL":
+            contacts = client.roster.getItems()
+            jids = [contact.jid.userhostJID() for contact in contacts]
+        elif publishers_type == "GROUP":
+            jids = []
+            for _group in publishers:
+                jids.extend(client.roster.getJidsFromGroup(_group))
+        elif publishers_type == 'JID':
+            jids = publishers
+        else:
+            raise UnknownType
+
+        data = {publisher: self.getNodeName(publisher) for publisher in jids}
+        d_dict = yield self.host.plugins["XEP-0060"].getItemsFromMany(client.item_access_pubsub, data, max_items=max_items, profile_key=profile)
+        for publisher, d in d_dict.items():
+            d.addCallback(self._itemsConstruction, publisher, client)
+            d.addCallback(lambda gbdata: (publisher.full(), gbdata))
+        # consume the failure "StanzaError with condition u'item-not-found'"
+        # when the node doesn't exist (e.g that JID hasn't posted any message)
+        result = yield defer.DeferredList(d_dict.values(), consumeErrors=True)
+        defer.returnValue({value[0]: value[1] for success, value in result if success})
+
     def subscribeGroupBlog(self, pub_jid, profile_key=C.PROF_KEY_NONE):
         def initialised(result):
             profile, client = result
@@ -721,46 +706,40 @@
             publishers_jids = publishers
         return self.massiveSubscribeGroupBlogs(publishers_type, publishers_jids, profile_key)
 
+    @defer.inlineCallbacks
     def massiveSubscribeGroupBlogs(self, publishers_type, publishers, profile_key=C.PROF_KEY_NONE):
         """Subscribe microblogs for a list of groups or jids
         @param publishers_type: type of the list of publishers (one of "GROUP" or "JID" or "ALL")
         @param publishers: list of publishers, according to "publishers_type" (list of groups or list of jids)
         @param profile_key: profile key
         """
-
-        def initialised(result):
-            profile, client = result
-
-            if publishers_type == "ALL":
-                contacts = client.roster.getItems()
-                jids = [contact.jid.userhostJID() for contact in contacts]
-            elif publishers_type == "GROUP":
-                jids = []
-                for _group in publishers:
-                    jids.extend(client.roster.getJidsFromGroup(_group))
-            elif publishers_type == 'JID':
-                jids = publishers
-            else:
-                raise UnknownType
-
-            mblogs = []
-            for jid_ in jids:
-                d = self.host.plugins["XEP-0060"].subscribe(client.item_access_pubsub, self.getNodeName(jid_),
-                                                            profile_key=profile_key)
-                mblogs.append(d)
-            # consume the failure "StanzaError with condition u'item-not-found'"
-            # when the node doesn't exist (e.g that JID hasn't posted any message)
-            dlist = defer.DeferredList(mblogs, consumeErrors=True)
-            return dlist
-
         #TODO: custom exception
         if publishers_type not in ["GROUP", "JID", "ALL"]:
             raise Exception("Bad call, unknown publishers_type")
         if publishers_type == "ALL" and publishers:
             raise Exception("Publishers list must be empty when getting microblogs for all contacts")
-        return self._initialise(profile_key).addCallback(initialised)
+        profile, client = yield self._initialise(profile_key)
         #TODO: we need to use the server corresponding the the host of the jid
 
+        if publishers_type == "ALL":
+            contacts = client.roster.getItems()
+            jids = [contact.jid.userhostJID() for contact in contacts]
+        elif publishers_type == "GROUP":
+            jids = []
+            for _group in publishers:
+                jids.extend(client.roster.getJidsFromGroup(_group))
+        elif publishers_type == 'JID':
+            jids = publishers
+        else:
+            raise UnknownType
+
+        node_ids = [self.getNodeName(publisher) for publisher in jids]
+        d_list = yield self.host.plugins["XEP-0060"].subscribeToMany(client.item_access_pubsub, node_ids, profile_key=profile_key)
+        # consume the failure "StanzaError with condition u'item-not-found'"
+        # when the node doesn't exist (e.g that JID hasn't posted any message)
+        result = yield defer.DeferredList(d_list, consumeErrors=True)
+        defer.returnValue(result)
+
     def deleteAllGroupBlogsAndComments(self, profile_key=C.PROF_KEY_NONE):
         """Delete absolutely all the microblog data that the user has posted"""
         calls = [self.deleteAllGroupBlogs(profile_key), self.deleteAllGroupBlogsComments(profile_key)]