Mercurial > libervia-backend
changeset 3308:384283adcce1
plugins XEP-0059, XEP-0060, XEP-0277, XEP-0313: better serialisation:
`data_format.serialise` is now used for `mbGet`, and RSM/MAM values are not transtyped to
strings anymore. A serialised dict is now used, items are put in the `items` key.
Comments handling has been refactored to use a list for the potentially multiple comments
nodes.
`rsm` data are now in a `rsm` key of the dict, and `mam` data are merged with other
metadata.
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Thu, 16 Jul 2020 09:07:20 +0200 |
| parents | 9f0e28137cd0 |
| children | 71761e9fb984 |
| files | sat/plugins/plugin_xep_0059.py sat/plugins/plugin_xep_0060.py sat/plugins/plugin_xep_0277.py sat/plugins/plugin_xep_0313.py sat_frontends/jp/cmd_blog.py |
| diffstat | 5 files changed, 152 insertions(+), 165 deletions(-) [+] |
line wrap: on
line diff
--- a/sat/plugins/plugin_xep_0059.py Fri Jun 19 15:47:16 2020 +0200 +++ b/sat/plugins/plugin_xep_0059.py Thu Jul 16 09:07:20 2020 +0200 @@ -77,8 +77,8 @@ else: return None - def serialise(self, rsm_response, data=None): - """Serialise data for RSM + def response2dict(self, rsm_response, data=None): + """Return a dict with RSM response Key set in data can be: - rsm_first: first item id in the page @@ -87,7 +87,7 @@ - rsm_count: total number of items in the full set (may be approximage) If a value doesn't exists, it's not set. All values are set as strings. - @param rsm_response(rsm.RSMResponse): response to serialise + @param rsm_response(rsm.RSMResponse): response to parse @param data(dict, None): dict to update with rsm_* data. If None, a new dict is created @return (dict): data dict @@ -95,13 +95,11 @@ if data is None: data = {} if rsm_response.first is not None: - data["rsm_first"] = rsm_response.first + data["first"] = rsm_response.first if rsm_response.last is not None: - data["rsm_last"] = rsm_response.last + data["last"] = rsm_response.last if rsm_response.index is not None: - data["rsm_index"] = str(rsm_response.index) - if rsm_response.index is not None: - data["rsm_index"] = str(rsm_response.index) + data["index"] = rsm_response.index return data
--- a/sat/plugins/plugin_xep_0060.py Fri Jun 19 15:47:16 2020 +0200 +++ b/sat/plugins/plugin_xep_0060.py Thu Jul 16 09:07:20 2020 +0200 @@ -717,16 +717,24 @@ "node": node, "uri": self.getNodeURI(service_jid, node), } + if mam_response is not None: + # mam_response is a dict with "complete" and "stable" keys + # we can put them directly in metadata + metadata.update(mam_response) if rsm_request is not None and rsm_response is not None: - metadata.update( - { - "rsm_" + key: value - for key, value in rsm_response.toDict().items() - } - ) - if mam_response is not None: - for key, value in mam_response.items(): - metadata["mam_" + key] = value + metadata['rsm'] = rsm_response.toDict() + if mam_response is None: + index = rsm_response.index + count = rsm_response.count + if index is None or count is None: + # we don't have enough information to know if the data is complete + # or not + metadata["complete"] = None + else: + # normally we have a strict equality here but XEP-0059 states + # that index MAY be approximative, so just in case… + metadata["complete"] = index + len(items) >= count + return (items, metadata) d.addCallback(addMetadata) @@ -1107,6 +1115,7 @@ @param node(unicode): node @return (unicode): URI of the node """ + # FIXME: deprecated, use sat.tools.common.uri instead assert service is not None # XXX: urllib.urlencode use "&" to separate value, while XMPP URL (cf. RFC 5122) # use ";" as a separator. So if more than one value is used in query_data, @@ -1127,66 +1136,40 @@ ): return self.rt_sessions.getResults(session_id, on_success, on_error, profile) - def transItemsData(self, items_data, item_cb=lambda item: item.toXml(), - serialise=False): + def transItemsData(self, items_data, item_cb=lambda item: item.toXml()): """Helper method to transform result from [getItems] the items_data must be a tuple(list[domish.Element], dict[unicode, unicode]) - as returned by [getItems]. metadata values are then casted to unicode and - each item is passed to items_cb then optionally serialised with - data_format.serialise. + as returned by [getItems]. @param items_data(tuple): tuple returned by [getItems] @param item_cb(callable): method to transform each item - @param serialise(bool): if True do a data_format.serialise - after applying item_cb @return (tuple): a serialised form ready to go throught bridge """ items, metadata = items_data - if serialise: - items = [data_format.serialise(item_cb(item)) for item in items] - else: - items = [item_cb(item) for item in items] + items = [item_cb(item) for item in items] - return ( - items, - {key: str(value) for key, value in metadata.items()}, - ) + return (items, metadata) - def transItemsDataD(self, items_data, item_cb, serialise=False): + def transItemsDataD(self, items_data, item_cb): """Helper method to transform result from [getItems], deferred version the items_data must be a tuple(list[domish.Element], dict[unicode, unicode]) as returned by [getItems]. metadata values are then casted to unicode and - each item is passed to items_cb then optionally serialised with - data_format.serialise. + each item is passed to items_cb. An errback is added to item_cb, and when it is fired the value is filtered from final items @param items_data(tuple): tuple returned by [getItems] @param item_cb(callable): method to transform each item (must return a deferred) - @param serialise(bool): if True do a data_format.serialise - after applying item_cb - @return (tuple): a deferred which fire a serialised form ready to go throught - bridge + @return (tuple): a deferred which fire a dict which can be serialised to go + throught bridge """ items, metadata = items_data def eb(failure_): - log.warning(f"Error while serialising/parsing item: {failure_.value}") + log.warning(f"Error while parsing item: {failure_.value}") d = defer.gatherResults([item_cb(item).addErrback(eb) for item in items]) - - def finishSerialisation(parsed_items): - if serialise: - items = [data_format.serialise(i) for i in parsed_items if i is not None] - else: - items = [i for i in parsed_items if i is not None] - - return ( - items, - {key: str(value) for key, value in metadata.items()}, - ) - - d.addCallback(finishSerialisation) + d.addCallback(lambda parsed_items: (parsed_items, metadata)) return d def serDList(self, results, failure_result=None):
--- a/sat/plugins/plugin_xep_0277.py Fri Jun 19 15:47:16 2020 +0200 +++ b/sat/plugins/plugin_xep_0277.py Thu Jul 16 09:07:20 2020 +0200 @@ -103,7 +103,7 @@ "mbGet", ".plugin", in_sign="ssiasa{ss}s", - out_sign="(asa{ss})", + out_sign="s", method=self._mbGet, async_=True, ) @@ -357,21 +357,25 @@ ) # links + comments = microblog_data['comments'] = [] for link_elt in entry_elt.elements(NS_ATOM, "link"): if ( link_elt.getAttribute("rel") == "replies" and link_elt.getAttribute("title") == "comments" ): - key = check_conflict("comments", True) - microblog_data[key] = link_elt["href"] + uri = link_elt["href"] + comments_data = { + "uri": uri, + } try: - service, node = self.parseCommentUrl(microblog_data[key]) + service, node = self.parseCommentUrl(uri) except Exception as e: - log.warning(f"Can't parse url {microblog_data[key]}: {e}") - del microblog_data[key] + log.warning(f"Can't parse comments url: {e}") + continue else: - microblog_data["{}_service".format(key)] = service.full() - microblog_data["{}_node".format(key)] = node + comments_data["service"] = service.full() + comments_data["node"] = node + comments.append(comments_data) else: rel = link_elt.getAttribute("rel", "") title = link_elt.getAttribute("title", "") @@ -589,9 +593,10 @@ entry_elt.addElement("id", content=entry_id) # ## comments ## - if "comments" in data: + for comments_data in data.get('comments', []): link_elt = entry_elt.addElement("link") - link_elt["href"] = data["comments"] + # XXX: "uri" is set in self._manageComments if not already existing + link_elt["href"] = comments_data["uri"] link_elt["rel"] = "replies" link_elt["title"] = "comments" @@ -646,8 +651,8 @@ def _manageComments(self, client, mb_data, service, node, item_id, access=None): """Check comments keys in mb_data and create comments node if necessary - if mb_data['comments'] exists, it is used (or mb_data['comments_service'] and/or mb_data['comments_node']), - else it is generated (if allow_comments is True). + if a comments node metadata is set in the mb_data['comments'] list, it is used + otherwise it is generated (if allow_comments is True). @param mb_data(dict): microblog mb_data @param service(jid.JID, None): PubSub service of the parent item @param node(unicode): node of the parent item @@ -655,11 +660,13 @@ @param access(unicode, None): access model None to use same access model as parent item """ - # FIXME: if 'comments' already exists in mb_data, - # it is not used to create the Node allow_comments = mb_data.pop("allow_comments", None) if allow_comments is None: - return + if "comments" in mb_data: + mb_data["allow_comments"] = True + else: + # no comments set or requested, nothing to do + return elif allow_comments == False: if "comments" in mb_data: log.warning( @@ -671,6 +678,13 @@ del mb_data["comments"] return + # we have usually a single comment node, but the spec allow several, so we need to + # handle this in a list + if len(mb_data.setdefault('comments', [])) == 0: + # we need at least one comment node + comments_data = {} + mb_data['comments'].append({}) + if access is None: # TODO: cache access models per service/node parent_node_config = yield self._p.getConfiguration(client, service, node) @@ -689,63 +703,65 @@ # if other plugins need to change the options yield self.host.trigger.point("XEP-0277_comments", client, mb_data, options) - try: - comments_node = mb_data["comments_node"] - except KeyError: - comments_node = self.getCommentsNode(item_id) - else: - if not comments_node: - raise exceptions.DataError( - "if comments_node is present, it must not be empty" + for comments_data in mb_data['comments']: + uri = comments_data.get('uri') + comments_node = comments_data.get('node') + try: + comments_service = jid.JID(comments_data["service"]) + except KeyError: + comments_service = None + + if uri: + uri_service, uri_node = self.parseCommentUrl(uri) + if ((comments_node is not None and comments_node!=uri_node) + or (comments_service is not None and comments_service!=uri_service)): + raise ValueError( + f"Incoherence between comments URI ({uri}) and comments_service " + f"({comments_service}) or comments_node ({comments_node})") + comments_data['service'] = comments_service = uri_service + comments_data['node'] = comments_node = uri_node + else: + if not comments_node: + comments_node = self.getCommentsNode(item_id) + comments_data['node'] = comments_node + if comments_service is None: + comments_service = yield self.getCommentsService(client, service) + if comments_service is None: + comments_service = client.jid.userhostJID() + comments_data['service'] = comments_service + + comments_data['uri'] = xmpp_uri.buildXMPPUri( + "pubsub", + path=comments_service.full(), + node=comments_node, ) - try: - comments_service = jid.JID(mb_data["comments_service"]) - except KeyError: - comments_service = yield self.getCommentsService(client, service) - - try: - yield self._p.createNode(client, comments_service, comments_node, options) - except error.StanzaError as e: - if e.condition == "conflict": - log.info( - "node {} already exists on service {}".format( - comments_node, comments_service + try: + yield self._p.createNode(client, comments_service, comments_node, options) + except error.StanzaError as e: + if e.condition == "conflict": + log.info( + "node {} already exists on service {}".format( + comments_node, comments_service + ) ) - ) + else: + raise e else: - raise e - else: - if access == self._p.ACCESS_WHITELIST: - # for whitelist access we need to copy affiliations from parent item - comments_affiliations = yield self._p.getNodeAffiliations( - client, service, node - ) - # …except for "member", that we transform to publisher - # because we wants members to be able to write to comments - for jid_, affiliation in list(comments_affiliations.items()): - if affiliation == "member": - comments_affiliations[jid_] == "publisher" + if access == self._p.ACCESS_WHITELIST: + # for whitelist access we need to copy affiliations from parent item + comments_affiliations = yield self._p.getNodeAffiliations( + client, service, node + ) + # …except for "member", that we transform to publisher + # because we wants members to be able to write to comments + for jid_, affiliation in list(comments_affiliations.items()): + if affiliation == "member": + comments_affiliations[jid_] == "publisher" - yield self._p.setNodeAffiliations( - client, comments_service, comments_node, comments_affiliations - ) - - if comments_service is None: - comments_service = client.jid.userhostJID() - - if "comments" in mb_data: - if not mb_data["comments"]: - raise exceptions.DataError( - "if comments is present, it must not be empty" - ) - if "comments_node" in mb_data or "comments_service" in mb_data: - raise exceptions.DataError( - "You can't use comments_service/comments_node and comments at the " - "same time" - ) - else: - mb_data["comments"] = self._p.getNodeURI(comments_service, comments_node) + yield self._p.setNodeAffiliations( + client, comments_service, comments_node, comments_affiliations + ) def _mbSend(self, service, node, data, profile_key): service = jid.JID(service) if service else None @@ -796,8 +812,8 @@ def _mbGetSerialise(self, data): items, metadata = data - items = [data_format.serialise(item) for item in items] - return items, metadata + metadata['items'] = items + return data_format.serialise(metadata) def _mbGet(self, service="", node="", max_items=10, item_ids=None, extra_dict=None, profile_key=C.PROF_KEY_NONE): @@ -840,7 +856,8 @@ rsm_request=rsm_request, extra=extra, ) - mb_data = yield self._p.transItemsDataD(items_data, self.item2mbdata) + mb_data = yield self._p.transItemsDataD( + items_data, self.item2mbdata) defer.returnValue(mb_data) def parseCommentUrl(self, node_url): @@ -1259,7 +1276,8 @@ client, service, node, max_items, rsm_request=rsm_request, extra=extra ) d.addCallback( - lambda items_data: self._p.transItemsDataD(items_data, self.item2mbdata) + lambda items_data: self._p.transItemsDataD( + items_data, self.item2mbdata) ) d.addCallback(getComments) d.addCallback(lambda items_comments_data: ("", items_comments_data))
--- a/sat/plugins/plugin_xep_0313.py Fri Jun 19 15:47:16 2020 +0200 +++ b/sat/plugins/plugin_xep_0313.py Thu Jul 16 09:07:20 2020 +0200 @@ -216,24 +216,6 @@ return mam.MAMRequest(**mam_args) if mam_args else None - def serialise(self, mam_response, data=None): - """Serialise data for MAM - - Key set in data can be: - - mam_complete: a bool const indicating if all items have been received - - mam_stable: a bool const which is False if items order may be changed - All values are set as strings. - @param mam_response(dict): response data to serialise - @param data(dict, None): dict to update with mam_* data. - If None, a new dict is created - @return (dict): data dict - """ - if data is None: - data = {} - data["mam_complete"] = C.boolConst(mam_response['complete']) - data["mam_stable"] = C.boolConst(mam_response['stable']) - return data - def getMessageFromResult(self, client, mess_elt, mam_req, service=None): """Extract usable <message/> from MAM query result @@ -345,8 +327,10 @@ service=service) mess_data = client.messageProt.parseMessage(fwd_message_elt) mess_list.append(client.messageGetBridgeArgs(mess_data)) - metadata = self._rsm.serialise(rsm_response) - self.serialise(mam_response, metadata) + metadata = { + 'rsm': self._rsm.response2dict(rsm_response), + 'mam': mam_response + } return mess_list, metadata, client.profile def _getArchives(self, service, extra_ser, profile_key): @@ -354,8 +338,8 @@ @return: tuple with: - list of message with same data as in bridge.messageNew - response metadata with: - - rsm data (rsm_first, rsm_last, rsm_count, rsm_index) - - mam data (mam_complete, mam_stable) + - rsm data (first, last, count, index) + - mam data (complete, stable) - profile """ client = self.host.getClient(profile_key)
--- a/sat_frontends/jp/cmd_blog.py Fri Jun 19 15:47:16 2020 +0200 +++ b/sat_frontends/jp/cmd_blog.py Thu Jul 16 09:07:20 2020 +0200 @@ -33,7 +33,6 @@ from sat_frontends.jp.constants import Const as C from sat_frontends.jp import common from sat.tools.common.ansi import ANSI as A -from sat.tools.common import data_objects from sat.tools.common import uri from sat.tools import config from configparser import NoSectionError, NoOptionError @@ -262,7 +261,9 @@ # TODO: add MAM filters def template_data_mapping(self, data): - return {"items": data_objects.BlogItems(data, deserialise=False)} + items, blog_items = data + blog_items['items'] = items + return {"blog_items": blog_items} def format_comments(self, item, keys): comments_data = data_format.dict2iterdict( @@ -424,22 +425,22 @@ async def start(self): try: - mb_result = await self.host.bridge.mbGet( - self.args.service, - self.args.node, - self.args.max, - self.args.items, - self.getPubsubExtra(), - self.profile + mb_data = data_format.deserialise( + await self.host.bridge.mbGet( + self.args.service, + self.args.node, + self.args.max, + self.args.items, + self.getPubsubExtra(), + self.profile + ) ) except Exception as e: self.disp(f"can't get blog items: {e}", error=True) self.host.quit(C.EXIT_BRIDGE_ERRBACK) else: - items, metadata = mb_result - items = [data_format.deserialise(i) for i in items] - mb_result = items, metadata - await self.output(mb_result) + items = mb_data.pop('items') + await self.output((items, mb_data)) self.host.quit(C.EXIT_OK) @@ -592,14 +593,17 @@ async def getItemData(self, service, node, item): items = [item] if item else [] - mb_data = await self.host.bridge.mbGet( - service, node, 1, items, {}, self.profile) - mb_data = data_format.deserialise(mb_data[0][0]) + mb_data = data_format.deserialise( + await self.host.bridge.mbGet( + service, node, 1, items, {}, self.profile + ) + ) + item = mb_data['items'][0] try: - content = mb_data["content_xhtml"] + content = item["content_xhtml"] except KeyError: - content = mb_data["content"] + content = item["content"] if content: content = await self.host.bridge.syntaxConvert( content, "text", SYNTAX_XHTML, False, self.profile @@ -623,7 +627,7 @@ root = etree.fromstring(content, parser) content = etree.tostring(root, encoding=str, pretty_print=True) - return content, mb_data, mb_data["id"] + return content, item, item["id"] async def start(self): # if there are user defined extension, we use them