comparison sat_pubsub/backend.py @ 430:5a0ada3b61ca

Full-Text Search implementation: /!\ pgsql schema needs to be updated /!\ /!\ Minimal PostgreSQL required version is now 12 /!\ A new options is available to specify main language of a node. By default a `generic` language is used (which uses the `simple` configuration in PostgreSQL). When a node owner changes the language, the index is rebuilt accordingly. It is possible to have item specific language for multilingual nodes (but for the moment the search is done with node language, so the results won't be good). If an item language is explicitely set in `item_languages`, the FTS configuration won't be affected by node FTS language setting. Search is parsed with `websearch_to_tsquery` for now, but this parser doesn't handle prefix matching, so it may be replaced in the future. SetConfiguration now only updates the modified values, this avoid triggering the FTS re-indexing on each config change. `_checkNodeExists` is not called anymore as we can check if a row has been modified to see if the node exists, this avoid a useless query. Item storing has been slighly improved with a useless SELECT and condition removed. To avoid 2 schema updates in a row, the `sat_pubsub_update_5_6.sql` file also prepares the implementation of XEP-0346 by updating nodes with a schema and creating the suitable template nodes.
author Goffi <goffi@goffi.org>
date Fri, 11 Dec 2020 17:18:52 +0100
parents c21f31355ab9
children 5e8b8ef5c862
comparison
equal deleted inserted replaced
429:0526073ff2ab 430:5a0ada3b61ca
80 80
81 from sat_pubsub import error 81 from sat_pubsub import error
82 from sat_pubsub import iidavoll 82 from sat_pubsub import iidavoll
83 from sat_pubsub import const 83 from sat_pubsub import const
84 from sat_pubsub import container 84 from sat_pubsub import container
85
86 # TODO: modernise the code, get rid of legacy Twisted logging, use coroutines,
87 # better formatting
85 88
86 89
87 def _getAffiliation(node, entity): 90 def _getAffiliation(node, entity):
88 d = node.getAffiliation(entity) 91 d = node.getAffiliation(entity)
89 d.addCallback(lambda affiliation: (node, affiliation)) 92 d.addCallback(lambda affiliation: (node, affiliation))
178 {"type": "boolean", 181 {"type": "boolean",
179 "label": "Use serial ids"}, 182 "label": "Use serial ids"},
180 const.OPT_CONSISTENT_PUBLISHER: 183 const.OPT_CONSISTENT_PUBLISHER:
181 {"type": "boolean", 184 {"type": "boolean",
182 "label": "Keep publisher on update"}, 185 "label": "Keep publisher on update"},
186 const.OPT_FTS_LANGUAGE:
187 {"type": "list-single",
188 "label": "Default language to use for full text search",
189 "options": {
190 const.VAL_FTS_GENERIC: (
191 "Generic (use if unsure of if your language is not present)"
192 )
193 }
194 },
183 } 195 }
184 196
185 subscriptionOptions = { 197 subscriptionOptions = {
186 "pubsub#subscription_type": 198 "pubsub#subscription_type":
187 {"type": "list-single", 199 {"type": "list-single",
201 utility.EventDispatcher.__init__(self) 213 utility.EventDispatcher.__init__(self)
202 self.storage = storage 214 self.storage = storage
203 self._callbackList = [] 215 self._callbackList = []
204 self.config = config 216 self.config = config
205 self.admins = config['admins_jids_list'] 217 self.admins = config['admins_jids_list']
218 d = self.storage.getFTSLanguages()
219 d.addCallbacks(self._getFTSLanguagesCb, self._getFTSLanguagesEb)
220
221 def _getFTSLanguagesCb(self, languages):
222 # we skip the first one which is always "generic", as we already have it
223 for l in languages[1:]:
224 self.nodeOptions[const.OPT_FTS_LANGUAGE]["options"][l] = l.title()
225
226 def _getFTSLanguagesEb(self, failure_):
227 log.msg(f"WARNING: can get FTS languages: {failure_}")
206 228
207 def isAdmin(self, entity_jid): 229 def isAdmin(self, entity_jid):
208 """Return True if an entity is an administrator""" 230 """Return True if an entity is an administrator"""
209 return entity_jid.userhostJID() in self.admins 231 return entity_jid.userhostJID() in self.admins
210 232
722 return d 744 return d
723 745
724 def setNodeSchema(self, nodeIdentifier, schema, requestor, pep, recipient): 746 def setNodeSchema(self, nodeIdentifier, schema, requestor, pep, recipient):
725 """set or remove Schema of a node 747 """set or remove Schema of a node
726 748
727 @param nodeIdentifier(unicode): identifier of the pubusb node 749 @param nodeIdentifier(unicode): identifier of the pubsub node
728 @param schema(domish.Element, None): schema to set 750 @param schema(domish.Element, None): schema to set
729 None to remove schema 751 None to remove schema
730 @param requestor(jid.JID): entity doing the request 752 @param requestor(jid.JID): entity doing the request
731 @param pep(bool): True if it's a PEP request 753 @param pep(bool): True if it's a PEP request
732 @param recipient(jid.JID, None): recipient of the PEP request 754 @param recipient(jid.JID, None): recipient of the PEP request