changeset 301:05c875a13a62

categories are now stored in a dedicated table if item contain an atom entry: - database creation/update files include the new table - item data are now stored in a ItemData namedtuple
author Goffi <goffi@goffi.org>
date Wed, 25 Nov 2015 18:33:38 +0100
parents c5acb4995fde
children b8b25efae0bc
files db/pubsub.sql db/sat_pubsub_update_0_1.sql sat_pubsub/backend.py sat_pubsub/const.py sat_pubsub/pgsql_storage.py
diffstat 5 files changed, 69 insertions(+), 13 deletions(-) [+]
line wrap: on
line diff
--- a/db/pubsub.sql	Wed Nov 25 16:00:08 2015 +0100
+++ b/db/pubsub.sql	Wed Nov 25 18:33:38 2015 +0100
@@ -73,6 +73,13 @@
     UNIQUE (item_id,groupname)
 );
 
+CREATE TABLE item_categories (
+    item_categories_id serial PRIMARY KEY,
+    item_id integer NOT NULL references items ON DELETE CASCADE,
+    category text NOT NULL,
+    UNIQUE (item_id,category)
+);
+
 CREATE TABLE metadata (
 	key text PRIMARY KEY,
 	value text
--- a/db/sat_pubsub_update_0_1.sql	Wed Nov 25 16:00:08 2015 +0100
+++ b/db/sat_pubsub_update_0_1.sql	Wed Nov 25 18:33:38 2015 +0100
@@ -12,6 +12,13 @@
 
 INSERT INTO metadata VALUES ('version', '1');
 
+CREATE TABLE item_categories (
+    item_categories_id serial PRIMARY KEY,
+    item_id integer NOT NULL references items ON DELETE CASCADE,
+    category text NOT NULL,
+    UNIQUE (item_id,category)
+);
+
 UPDATE nodes SET node='urn:xmpp:microblog:0', pep=substring(node from 20) WHERE node LIKE 'urn:xmpp:groupblog:_%';
 
 /* This is to update namespaces, SàT was bugguy before 0.6 and didn't set the atom namespace in <entry/> */
--- a/sat_pubsub/backend.py	Wed Nov 25 16:00:08 2015 +0100
+++ b/sat_pubsub/backend.py	Wed Nov 25 18:33:38 2015 +0100
@@ -81,6 +81,10 @@
 from sat_pubsub.iidavoll import IBackendService, ILeafNode
 
 from copy import deepcopy
+from collections import namedtuple
+
+
+ItemData = namedtuple('ItemData', ('item', 'access_model', 'config', 'categories'))
 
 
 def _getAffiliation(node, entity):
@@ -245,7 +249,9 @@
 
     def parseItemConfig(self, item):
         """Get and remove item configuration information
-        @param item:
+
+        @param item (domish.Element): item to parse
+        @return (tuple[unicode, dict)): (access_model, item_config)
         """
         item_config = None
         access_model = const.VAL_AMODEL_DEFAULT
@@ -262,6 +268,25 @@
             access_model = item_config.get(const.OPT_ACCESS_MODEL, const.VAL_AMODEL_DEFAULT)
         return (access_model, item_config)
 
+    def parseCategories(self, item_elt):
+        """Check if item contain an atom entry, and parse categories if possible
+
+        @param item (domish.Element): item to parse
+        @return (list): list of found categories
+        """
+        categories = []
+        try:
+            entry_elt = item_elt.elements(const.NS_ATOM, "entry").next()
+        except StopIteration:
+            return categories
+
+        for category_elt in entry_elt.elements(const.NS_ATOM, 'category'):
+            category = category_elt.getAttribute('term')
+            if category:
+                categories.append(category)
+
+        return categories
+
     def _checkOverwrite(self, node, itemIdentifiers, publisher):
         """Check that the itemIdentifiers correspond to items published
         by the current publisher"""
@@ -311,7 +336,8 @@
                 else:
                     check_overwrite = True
             access_model, item_config = self.parseItemConfig(item)
-            items_data.append((item, access_model, item_config))
+            categories = self.parseCategories(item)
+            items_data.append(ItemData(item, access_model, item_config, categories))
 
         if persistItems:
             if check_overwrite and affiliation != 'owner':
@@ -331,8 +357,8 @@
 
     def _doNotify(self, result, node, items_data, deliverPayloads, pep, recipient):
         if items_data and not deliverPayloads:
-            for access_model, item_config, item in items_data:
-                item.children = []
+            for item_data in items_data:
+                item_data.item.children = []
         self.dispatch({'items_data': items_data, 'node': node, 'pep': pep, 'recipient': recipient},
                       '//event/pubsub/notify')
 
@@ -575,8 +601,7 @@
 
         def append_item_config(items_data):
             ret = []
-            for data in items_data:
-                item, access_model, access_list = data
+            for item, access_model, access_list in items_data:
                 if access_model == const.VAL_AMODEL_OPEN:
                     pass
                 elif access_model == const.VAL_AMODEL_ROSTER:
@@ -736,7 +761,7 @@
         def removeItems(items_data):
             """Remove the items and keep only actually removed ones in items_data"""
             d = node.removeItems(itemIdentifiers)
-            d.addCallback(lambda removed: [item_data for item_data in items_data if item_data[0]["id"] in removed])
+            d.addCallback(lambda removed: [item_data for item_data in items_data if item_data.item["id"] in removed])
             return d
 
         d = node.getItemsById(None, True, itemIdentifiers)
@@ -945,7 +970,7 @@
                 """
                 #TODO: a test should check that only the owner get the item configuration back
 
-                item, access_model, item_config = item_data
+                item, item_config = item_data.item, item_data.config
                 new_item = deepcopy(item)
                 if item_config:
                     new_item.addChild(item_config.toElement())
@@ -1031,14 +1056,16 @@
             #we filter items not allowed for the subscribers
             notifications_filtered = []
 
-            for subscriber, subscriptions, _items_data in notifications:
+            for subscriber, subscriptions, items_data in notifications:
                 if subscriber == owner_jid:
                     # as notification is always sent to owner,
                     # we ignore owner if he is here
                     continue
                 allowed_items = [] #we keep only item which subscriber can access
 
-                for item, access_model, access_list in _items_data:
+                for item_data in items_data:
+                    item, access_model = item_data.item, item_data.access_model
+                    access_list = item_data.config
                     if access_model == const.VAL_AMODEL_OPEN:
                         allowed_items.append(item)
                     elif access_model == const.VAL_AMODEL_ROSTER:
--- a/sat_pubsub/const.py	Wed Nov 25 16:00:08 2015 +0100
+++ b/sat_pubsub/const.py	Wed Nov 25 18:33:38 2015 +0100
@@ -55,6 +55,7 @@
 
 NS_GROUPBLOG_PREFIX = 'urn:xmpp:groupblog:'
 NS_ITEM_CONFIG = "http://jabber.org/protocol/pubsub#item-config"
+NS_ATOM = "http://www.w3.org/2005/Atom"
 OPT_ACCESS_MODEL = 'pubsub#access_model'
 OPT_ROSTER_GROUPS_ALLOWED = 'pubsub#roster_groups_allowed'
 OPT_PERSIST_ITEMS = "pubsub#persist_items"
--- a/sat_pubsub/pgsql_storage.py	Wed Nov 25 16:00:08 2015 +0100
+++ b/sat_pubsub/pgsql_storage.py	Wed Nov 25 18:33:38 2015 +0100
@@ -608,18 +608,21 @@
 
 
     def _storeItem(self, cursor, item_data, publisher):
-        item, access_model, item_config = item_data
+        item, access_model, item_config = item_data.item, item_data.access_model, item_data.config
         data = item.toXml()
 
         cursor.execute("""UPDATE items SET date=now(), publisher=%s, data=%s
                           FROM nodes
                           WHERE nodes.node_id = items.node_id AND
-                                nodes.node_id = %s and items.item=%s""",
+                                nodes.node_id = %s and items.item=%s
+                          RETURNING item_id""",
                        (publisher.full(),
                         data,
                         self.nodeDbId,
                         item["id"]))
         if cursor.rowcount == 1:
+            item_id = cursor.fetchone()[0];
+            self._storeCategories(cursor, item_id, item_data.categories, update=True)
             return
 
         cursor.execute("""INSERT INTO items (node_id, item, publisher, data, access_model)
@@ -633,8 +636,10 @@
                         access_model,
                         self.nodeDbId))
 
+        item_id = cursor.fetchone()[0];
+        self._storeCategories(cursor, item_id, item_data.categories)
+
         if access_model == const.VAL_AMODEL_ROSTER:
-            item_id = cursor.fetchone()[0];
             if const.OPT_ROSTER_GROUPS_ALLOWED in item_config:
                 item_config.fields[const.OPT_ROSTER_GROUPS_ALLOWED].fieldType='list-multi' #XXX: needed to force list if there is only one value
                 allowed_groups = item_config[const.OPT_ROSTER_GROUPS_ALLOWED]
@@ -645,6 +650,15 @@
                 cursor.execute("""INSERT INTO item_groups_authorized (item_id, groupname)
                                   VALUES (%s,%s)""" , (item_id, group))
 
+    def _storeCategories(self, cursor, item_id, categories, update=False):
+        # TODO: handle canonical form
+        if update:
+            cursor.execute("""DELETE FROM item_categories
+                              WHERE item_id=%s""", (item_id,))
+
+        for category in categories:
+            cursor.execute("""INSERT INTO item_categories (item_id, category)
+                              VALUES (%s, %s)""", (item_id, category))
 
     def removeItems(self, itemIdentifiers):
         return self.dbpool.runInteraction(self._removeItems, itemIdentifiers)