changeset 3308:384283adcce1

plugins XEP-0059, XEP-0060, XEP-0277, XEP-0313: better serialisation: `data_format.serialise` is now used for `mbGet`, and RSM/MAM values are not transtyped to strings anymore. A serialised dict is now used, items are put in the `items` key. Comments handling has been refactored to use a list for the potentially multiple comments nodes. `rsm` data are now in a `rsm` key of the dict, and `mam` data are merged with other metadata.
author Goffi <goffi@goffi.org>
date Thu, 16 Jul 2020 09:07:20 +0200 (2020-07-16)
parents 9f0e28137cd0
children 71761e9fb984
files sat/plugins/plugin_xep_0059.py sat/plugins/plugin_xep_0060.py sat/plugins/plugin_xep_0277.py sat/plugins/plugin_xep_0313.py sat_frontends/jp/cmd_blog.py
diffstat 5 files changed, 152 insertions(+), 165 deletions(-) [+]
line wrap: on
line diff
--- a/sat/plugins/plugin_xep_0059.py	Fri Jun 19 15:47:16 2020 +0200
+++ b/sat/plugins/plugin_xep_0059.py	Thu Jul 16 09:07:20 2020 +0200
@@ -77,8 +77,8 @@
         else:
             return None
 
-    def serialise(self, rsm_response, data=None):
-        """Serialise data for RSM
+    def response2dict(self, rsm_response, data=None):
+        """Return a dict with RSM response
 
         Key set in data can be:
             - rsm_first: first item id in the page
@@ -87,7 +87,7 @@
             - rsm_count: total number of items in the full set (may be approximage)
         If a value doesn't exists, it's not set.
         All values are set as strings.
-        @param rsm_response(rsm.RSMResponse): response to serialise
+        @param rsm_response(rsm.RSMResponse): response to parse
         @param data(dict, None): dict to update with rsm_* data.
             If None, a new dict is created
         @return (dict): data dict
@@ -95,13 +95,11 @@
         if data is None:
             data = {}
         if rsm_response.first is not None:
-            data["rsm_first"] = rsm_response.first
+            data["first"] = rsm_response.first
         if rsm_response.last is not None:
-            data["rsm_last"] = rsm_response.last
+            data["last"] = rsm_response.last
         if rsm_response.index is not None:
-            data["rsm_index"] = str(rsm_response.index)
-        if rsm_response.index is not None:
-            data["rsm_index"] = str(rsm_response.index)
+            data["index"] = rsm_response.index
         return data
 
 
--- a/sat/plugins/plugin_xep_0060.py	Fri Jun 19 15:47:16 2020 +0200
+++ b/sat/plugins/plugin_xep_0060.py	Thu Jul 16 09:07:20 2020 +0200
@@ -717,16 +717,24 @@
                 "node": node,
                 "uri": self.getNodeURI(service_jid, node),
             }
+            if mam_response is not None:
+                # mam_response is a dict with "complete" and "stable" keys
+                # we can put them directly in metadata
+                metadata.update(mam_response)
             if rsm_request is not None and rsm_response is not None:
-                metadata.update(
-                    {
-                        "rsm_" + key: value
-                        for key, value in rsm_response.toDict().items()
-                    }
-                )
-            if mam_response is not None:
-                for key, value in mam_response.items():
-                    metadata["mam_" + key] = value
+                metadata['rsm'] = rsm_response.toDict()
+                if mam_response is None:
+                    index = rsm_response.index
+                    count = rsm_response.count
+                    if index is None or count is None:
+                        # we don't have enough information to know if the data is complete
+                        # or not
+                        metadata["complete"] = None
+                    else:
+                        # normally we have a strict equality here but XEP-0059 states
+                        # that index MAY be approximative, so just in case…
+                        metadata["complete"] = index + len(items) >= count
+
             return (items, metadata)
 
         d.addCallback(addMetadata)
@@ -1107,6 +1115,7 @@
         @param node(unicode): node
         @return (unicode): URI of the node
         """
+        # FIXME: deprecated, use sat.tools.common.uri instead
         assert service is not None
         # XXX: urllib.urlencode use "&" to separate value, while XMPP URL (cf. RFC 5122)
         #      use ";" as a separator. So if more than one value is used in query_data,
@@ -1127,66 +1136,40 @@
     ):
         return self.rt_sessions.getResults(session_id, on_success, on_error, profile)
 
-    def transItemsData(self, items_data, item_cb=lambda item: item.toXml(),
-            serialise=False):
+    def transItemsData(self, items_data, item_cb=lambda item: item.toXml()):
         """Helper method to transform result from [getItems]
 
         the items_data must be a tuple(list[domish.Element], dict[unicode, unicode])
-        as returned by [getItems]. metadata values are then casted to unicode and
-        each item is passed to items_cb then optionally serialised with
-            data_format.serialise.
+        as returned by [getItems].
         @param items_data(tuple): tuple returned by [getItems]
         @param item_cb(callable): method to transform each item
-        @param serialise(bool): if True do a data_format.serialise
-            after applying item_cb
         @return (tuple): a serialised form ready to go throught bridge
         """
         items, metadata = items_data
-        if serialise:
-            items = [data_format.serialise(item_cb(item)) for item in items]
-        else:
-            items = [item_cb(item) for item in items]
+        items = [item_cb(item) for item in items]
 
-        return (
-            items,
-            {key: str(value) for key, value in metadata.items()},
-        )
+        return (items, metadata)
 
-    def transItemsDataD(self, items_data, item_cb, serialise=False):
+    def transItemsDataD(self, items_data, item_cb):
         """Helper method to transform result from [getItems], deferred version
 
         the items_data must be a tuple(list[domish.Element], dict[unicode, unicode])
         as returned by [getItems]. metadata values are then casted to unicode and
-        each item is passed to items_cb then optionally serialised with
-            data_format.serialise.
+        each item is passed to items_cb.
         An errback is added to item_cb, and when it is fired the value is filtered from
             final items
         @param items_data(tuple): tuple returned by [getItems]
         @param item_cb(callable): method to transform each item (must return a deferred)
-        @param serialise(bool): if True do a data_format.serialise
-            after applying item_cb
-        @return (tuple): a deferred which fire a serialised form ready to go throught
-            bridge
+        @return (tuple): a deferred which fire a dict which can be serialised to go
+            throught bridge
         """
         items, metadata = items_data
 
         def eb(failure_):
-            log.warning(f"Error while serialising/parsing item: {failure_.value}")
+            log.warning(f"Error while parsing item: {failure_.value}")
 
         d = defer.gatherResults([item_cb(item).addErrback(eb) for item in items])
-
-        def finishSerialisation(parsed_items):
-            if serialise:
-                items = [data_format.serialise(i) for i in parsed_items if i is not None]
-            else:
-                items = [i for i in parsed_items if i is not None]
-
-            return (
-                items,
-                {key: str(value) for key, value in metadata.items()},
-            )
-
-        d.addCallback(finishSerialisation)
+        d.addCallback(lambda parsed_items: (parsed_items, metadata))
         return d
 
     def serDList(self, results, failure_result=None):
--- a/sat/plugins/plugin_xep_0277.py	Fri Jun 19 15:47:16 2020 +0200
+++ b/sat/plugins/plugin_xep_0277.py	Thu Jul 16 09:07:20 2020 +0200
@@ -103,7 +103,7 @@
             "mbGet",
             ".plugin",
             in_sign="ssiasa{ss}s",
-            out_sign="(asa{ss})",
+            out_sign="s",
             method=self._mbGet,
             async_=True,
         )
@@ -357,21 +357,25 @@
             )
 
         # links
+        comments = microblog_data['comments'] = []
         for link_elt in entry_elt.elements(NS_ATOM, "link"):
             if (
                 link_elt.getAttribute("rel") == "replies"
                 and link_elt.getAttribute("title") == "comments"
             ):
-                key = check_conflict("comments", True)
-                microblog_data[key] = link_elt["href"]
+                uri = link_elt["href"]
+                comments_data = {
+                    "uri": uri,
+                }
                 try:
-                    service, node = self.parseCommentUrl(microblog_data[key])
+                    service, node = self.parseCommentUrl(uri)
                 except Exception as e:
-                    log.warning(f"Can't parse url {microblog_data[key]}: {e}")
-                    del microblog_data[key]
+                    log.warning(f"Can't parse comments url: {e}")
+                    continue
                 else:
-                    microblog_data["{}_service".format(key)] = service.full()
-                    microblog_data["{}_node".format(key)] = node
+                    comments_data["service"] = service.full()
+                    comments_data["node"] = node
+                comments.append(comments_data)
             else:
                 rel = link_elt.getAttribute("rel", "")
                 title = link_elt.getAttribute("title", "")
@@ -589,9 +593,10 @@
         entry_elt.addElement("id", content=entry_id)  #
 
         ## comments ##
-        if "comments" in data:
+        for comments_data in data.get('comments', []):
             link_elt = entry_elt.addElement("link")
-            link_elt["href"] = data["comments"]
+            # XXX: "uri" is set in self._manageComments if not already existing
+            link_elt["href"] = comments_data["uri"]
             link_elt["rel"] = "replies"
             link_elt["title"] = "comments"
 
@@ -646,8 +651,8 @@
     def _manageComments(self, client, mb_data, service, node, item_id, access=None):
         """Check comments keys in mb_data and create comments node if necessary
 
-        if mb_data['comments'] exists, it is used (or mb_data['comments_service'] and/or mb_data['comments_node']),
-        else it is generated (if allow_comments is True).
+        if a comments node metadata is set in the mb_data['comments'] list, it is used
+        otherwise it is generated (if allow_comments is True).
         @param mb_data(dict): microblog mb_data
         @param service(jid.JID, None): PubSub service of the parent item
         @param node(unicode): node of the parent item
@@ -655,11 +660,13 @@
         @param access(unicode, None): access model
             None to use same access model as parent item
         """
-        # FIXME: if 'comments' already exists in mb_data,
-        #        it is not used to create the Node
         allow_comments = mb_data.pop("allow_comments", None)
         if allow_comments is None:
-            return
+            if "comments" in mb_data:
+                mb_data["allow_comments"] = True
+            else:
+                # no comments set or requested, nothing to do
+                return
         elif allow_comments == False:
             if "comments" in mb_data:
                 log.warning(
@@ -671,6 +678,13 @@
                 del mb_data["comments"]
             return
 
+        # we have usually a single comment node, but the spec allow several, so we need to
+        # handle this in a list
+        if len(mb_data.setdefault('comments', [])) == 0:
+            # we need at least one comment node
+            comments_data = {}
+            mb_data['comments'].append({})
+
         if access is None:
             # TODO: cache access models per service/node
             parent_node_config = yield self._p.getConfiguration(client, service, node)
@@ -689,63 +703,65 @@
         # if other plugins need to change the options
         yield self.host.trigger.point("XEP-0277_comments", client, mb_data, options)
 
-        try:
-            comments_node = mb_data["comments_node"]
-        except KeyError:
-            comments_node = self.getCommentsNode(item_id)
-        else:
-            if not comments_node:
-                raise exceptions.DataError(
-                    "if comments_node is present, it must not be empty"
+        for comments_data in mb_data['comments']:
+            uri = comments_data.get('uri')
+            comments_node = comments_data.get('node')
+            try:
+                comments_service = jid.JID(comments_data["service"])
+            except KeyError:
+                comments_service = None
+
+            if uri:
+                uri_service, uri_node = self.parseCommentUrl(uri)
+                if ((comments_node is not None and comments_node!=uri_node)
+                     or (comments_service is not None and comments_service!=uri_service)):
+                    raise ValueError(
+                        f"Incoherence between comments URI ({uri}) and comments_service "
+                        f"({comments_service}) or comments_node ({comments_node})")
+                comments_data['service'] = comments_service = uri_service
+                comments_data['node'] = comments_node = uri_node
+            else:
+                if not comments_node:
+                    comments_node = self.getCommentsNode(item_id)
+                comments_data['node'] = comments_node
+                if comments_service is None:
+                    comments_service = yield self.getCommentsService(client, service)
+                    if comments_service is None:
+                        comments_service = client.jid.userhostJID()
+                comments_data['service'] = comments_service
+
+                comments_data['uri'] = xmpp_uri.buildXMPPUri(
+                    "pubsub",
+                    path=comments_service.full(),
+                    node=comments_node,
                 )
 
-        try:
-            comments_service = jid.JID(mb_data["comments_service"])
-        except KeyError:
-            comments_service = yield self.getCommentsService(client, service)
-
-        try:
-            yield self._p.createNode(client, comments_service, comments_node, options)
-        except error.StanzaError as e:
-            if e.condition == "conflict":
-                log.info(
-                    "node {} already exists on service {}".format(
-                        comments_node, comments_service
+            try:
+                yield self._p.createNode(client, comments_service, comments_node, options)
+            except error.StanzaError as e:
+                if e.condition == "conflict":
+                    log.info(
+                        "node {} already exists on service {}".format(
+                            comments_node, comments_service
+                        )
                     )
-                )
+                else:
+                    raise e
             else:
-                raise e
-        else:
-            if access == self._p.ACCESS_WHITELIST:
-                # for whitelist access we need to copy affiliations from parent item
-                comments_affiliations = yield self._p.getNodeAffiliations(
-                    client, service, node
-                )
-                # …except for "member", that we transform to publisher
-                # because we wants members to be able to write to comments
-                for jid_, affiliation in list(comments_affiliations.items()):
-                    if affiliation == "member":
-                        comments_affiliations[jid_] == "publisher"
+                if access == self._p.ACCESS_WHITELIST:
+                    # for whitelist access we need to copy affiliations from parent item
+                    comments_affiliations = yield self._p.getNodeAffiliations(
+                        client, service, node
+                    )
+                    # …except for "member", that we transform to publisher
+                    # because we wants members to be able to write to comments
+                    for jid_, affiliation in list(comments_affiliations.items()):
+                        if affiliation == "member":
+                            comments_affiliations[jid_] == "publisher"
 
-                yield self._p.setNodeAffiliations(
-                    client, comments_service, comments_node, comments_affiliations
-                )
-
-        if comments_service is None:
-            comments_service = client.jid.userhostJID()
-
-        if "comments" in mb_data:
-            if not mb_data["comments"]:
-                raise exceptions.DataError(
-                    "if comments is present, it must not be empty"
-                )
-            if "comments_node" in mb_data or "comments_service" in mb_data:
-                raise exceptions.DataError(
-                    "You can't use comments_service/comments_node and comments at the "
-                    "same time"
-                )
-        else:
-            mb_data["comments"] = self._p.getNodeURI(comments_service, comments_node)
+                    yield self._p.setNodeAffiliations(
+                        client, comments_service, comments_node, comments_affiliations
+                    )
 
     def _mbSend(self, service, node, data, profile_key):
         service = jid.JID(service) if service else None
@@ -796,8 +812,8 @@
 
     def _mbGetSerialise(self, data):
         items, metadata = data
-        items = [data_format.serialise(item) for item in items]
-        return items, metadata
+        metadata['items'] = items
+        return data_format.serialise(metadata)
 
     def _mbGet(self, service="", node="", max_items=10, item_ids=None, extra_dict=None,
                profile_key=C.PROF_KEY_NONE):
@@ -840,7 +856,8 @@
             rsm_request=rsm_request,
             extra=extra,
         )
-        mb_data = yield self._p.transItemsDataD(items_data, self.item2mbdata)
+        mb_data = yield self._p.transItemsDataD(
+            items_data, self.item2mbdata)
         defer.returnValue(mb_data)
 
     def parseCommentUrl(self, node_url):
@@ -1259,7 +1276,8 @@
                 client, service, node, max_items, rsm_request=rsm_request, extra=extra
             )
             d.addCallback(
-                lambda items_data: self._p.transItemsDataD(items_data, self.item2mbdata)
+                lambda items_data: self._p.transItemsDataD(
+                    items_data, self.item2mbdata)
             )
             d.addCallback(getComments)
             d.addCallback(lambda items_comments_data: ("", items_comments_data))
--- a/sat/plugins/plugin_xep_0313.py	Fri Jun 19 15:47:16 2020 +0200
+++ b/sat/plugins/plugin_xep_0313.py	Thu Jul 16 09:07:20 2020 +0200
@@ -216,24 +216,6 @@
 
         return mam.MAMRequest(**mam_args) if mam_args else None
 
-    def serialise(self, mam_response, data=None):
-        """Serialise data for MAM
-
-        Key set in data can be:
-            - mam_complete: a bool const indicating if all items have been received
-            - mam_stable: a bool const which is False if items order may be changed
-        All values are set as strings.
-        @param mam_response(dict): response data to serialise
-        @param data(dict, None): dict to update with mam_* data.
-            If None, a new dict is created
-        @return (dict): data dict
-        """
-        if data is None:
-            data = {}
-        data["mam_complete"] = C.boolConst(mam_response['complete'])
-        data["mam_stable"] = C.boolConst(mam_response['stable'])
-        return data
-
     def getMessageFromResult(self, client, mess_elt, mam_req, service=None):
         """Extract usable <message/> from MAM query result
 
@@ -345,8 +327,10 @@
                                                         service=service)
             mess_data = client.messageProt.parseMessage(fwd_message_elt)
             mess_list.append(client.messageGetBridgeArgs(mess_data))
-        metadata = self._rsm.serialise(rsm_response)
-        self.serialise(mam_response, metadata)
+        metadata = {
+            'rsm': self._rsm.response2dict(rsm_response),
+            'mam': mam_response
+        }
         return mess_list, metadata, client.profile
 
     def _getArchives(self, service, extra_ser, profile_key):
@@ -354,8 +338,8 @@
         @return: tuple with:
             - list of message with same data as in bridge.messageNew
             - response metadata with:
-                - rsm data (rsm_first, rsm_last, rsm_count, rsm_index)
-                - mam data (mam_complete, mam_stable)
+                - rsm data (first, last, count, index)
+                - mam data (complete, stable)
             - profile
         """
         client = self.host.getClient(profile_key)
--- a/sat_frontends/jp/cmd_blog.py	Fri Jun 19 15:47:16 2020 +0200
+++ b/sat_frontends/jp/cmd_blog.py	Thu Jul 16 09:07:20 2020 +0200
@@ -33,7 +33,6 @@
 from sat_frontends.jp.constants import Const as C
 from sat_frontends.jp import common
 from sat.tools.common.ansi import ANSI as A
-from sat.tools.common import data_objects
 from sat.tools.common import uri
 from sat.tools import config
 from configparser import NoSectionError, NoOptionError
@@ -262,7 +261,9 @@
         # TODO: add MAM filters
 
     def template_data_mapping(self, data):
-        return {"items": data_objects.BlogItems(data, deserialise=False)}
+        items, blog_items = data
+        blog_items['items'] = items
+        return {"blog_items": blog_items}
 
     def format_comments(self, item, keys):
         comments_data = data_format.dict2iterdict(
@@ -424,22 +425,22 @@
 
     async def start(self):
         try:
-            mb_result = await self.host.bridge.mbGet(
-                self.args.service,
-                self.args.node,
-                self.args.max,
-                self.args.items,
-                self.getPubsubExtra(),
-                self.profile
+            mb_data = data_format.deserialise(
+                await self.host.bridge.mbGet(
+                    self.args.service,
+                    self.args.node,
+                    self.args.max,
+                    self.args.items,
+                    self.getPubsubExtra(),
+                    self.profile
+                )
             )
         except Exception as e:
             self.disp(f"can't get blog items: {e}", error=True)
             self.host.quit(C.EXIT_BRIDGE_ERRBACK)
         else:
-            items, metadata = mb_result
-            items = [data_format.deserialise(i) for i in items]
-            mb_result = items, metadata
-            await self.output(mb_result)
+            items = mb_data.pop('items')
+            await self.output((items, mb_data))
             self.host.quit(C.EXIT_OK)
 
 
@@ -592,14 +593,17 @@
     async def getItemData(self, service, node, item):
         items = [item] if item else []
 
-        mb_data = await self.host.bridge.mbGet(
-            service, node, 1, items, {}, self.profile)
-        mb_data = data_format.deserialise(mb_data[0][0])
+        mb_data = data_format.deserialise(
+            await self.host.bridge.mbGet(
+                service, node, 1, items, {}, self.profile
+            )
+        )
+        item = mb_data['items'][0]
 
         try:
-            content = mb_data["content_xhtml"]
+            content = item["content_xhtml"]
         except KeyError:
-            content = mb_data["content"]
+            content = item["content"]
             if content:
                 content = await self.host.bridge.syntaxConvert(
                     content, "text", SYNTAX_XHTML, False, self.profile
@@ -623,7 +627,7 @@
                 root = etree.fromstring(content, parser)
                 content = etree.tostring(root, encoding=str, pretty_print=True)
 
-        return content, mb_data, mb_data["id"]
+        return content, item, item["id"]
 
     async def start(self):
         # if there are user defined extension, we use them