# HG changeset patch # User Goffi # Date 1468795922 -7200 # Node ID b536dd121da13bbd599fe2797f09987973a283e8 # Parent 53587e738aca1519cc3cbb2bf18fac7fcaaaff95 backend (memory), frontends: improved history filtering: a "filters" dictionnary is now use to filter, it can have, for now, filtering on: - "body": filter only on the body (equivalent to former "search" parameter, but not case sensitive) - "search": fitler on body + source resource - "types": allowed types - "not_types": forbidden types primitivus now do searching using "search", i.e. source resource is now taken into account (and search is now case insensitive) diff -r 53587e738aca -r b536dd121da1 frontends/src/bridge/DBus.py --- a/frontends/src/bridge/DBus.py Sun Jul 17 19:44:15 2016 +0200 +++ b/frontends/src/bridge/DBus.py Mon Jul 18 00:52:02 2016 +0200 @@ -508,14 +508,14 @@ kwargs['error_handler'] = error_handler return self.db_core_iface.getWaitingSub(profile_key, **kwargs) - def historyGet(self, from_jid, to_jid, limit, between=True, search='', profile="@NONE@", callback=None, errback=None): + def historyGet(self, from_jid, to_jid, limit, between=True, filters='', profile="@NONE@", callback=None, errback=None): if callback is None: error_handler = None else: if errback is None: errback = log.error error_handler = lambda err:errback(dbus_to_bridge_exception(err)) - return self.db_core_iface.historyGet(from_jid, to_jid, limit, between, search, profile, timeout=const_TIMEOUT, reply_handler=callback, error_handler=error_handler) + return self.db_core_iface.historyGet(from_jid, to_jid, limit, between, filters, profile, timeout=const_TIMEOUT, reply_handler=callback, error_handler=error_handler) def isConnected(self, profile_key="@DEFAULT@", callback=None, errback=None): if callback is None: diff -r 53587e738aca -r b536dd121da1 frontends/src/primitivus/chat.py --- a/frontends/src/primitivus/chat.py Sun Jul 17 19:44:15 2016 +0200 +++ b/frontends/src/primitivus/chat.py Mon Jul 18 00:52:02 2016 +0200 @@ -540,12 +540,12 @@ except AttributeError: pass - def updateHistory(self, size=C.HISTORY_LIMIT_DEFAULT, search='', profile='@NONE@'): + def updateHistory(self, size=C.HISTORY_LIMIT_DEFAULT, filters=None, profile='@NONE@'): del self.mess_walker[:] - if search: - self.mess_walker.append(urwid.Text(_(u"Results for searching the globbing pattern: {}").format(search))) - self.mess_walker.append(urwid.Text(_(u"Type ':history ' to reset the chat history").format(search))) - super(Chat, self).updateHistory(size, search, profile) + if filters and 'search' in filters: + self.mess_walker.append(urwid.Text(_(u"Results for searching the globbing pattern: {}").format(filters['search']))) + self.mess_walker.append(urwid.Text(_(u"Type ':history ' to reset the chat history"))) + super(Chat, self).updateHistory(size, filters, profile) def _onHistoryPrinted(self): """Refresh or scroll down the focus after the history is printed""" diff -r 53587e738aca -r b536dd121da1 frontends/src/primitivus/primitivus --- a/frontends/src/primitivus/primitivus Sun Jul 17 19:44:15 2016 +0200 +++ b/frontends/src/primitivus/primitivus Mon Jul 18 00:52:02 2016 +0200 @@ -130,7 +130,7 @@ if not pattern: self.host.notif_bar.addMessage(D_("Please specify the globbing pattern to search for")) else: - widget.updateHistory(size=C.HISTORY_LIMIT_NONE, search=pattern, profile=widget.profile) + widget.updateHistory(size=C.HISTORY_LIMIT_NONE, filters={'search': pattern}, profile=widget.profile) elif command == 'filter': # FIXME: filter is now only for current widget, # need to be able to set it globally or per widget diff -r 53587e738aca -r b536dd121da1 frontends/src/quick_frontend/quick_chat.py --- a/frontends/src/quick_frontend/quick_chat.py Sun Jul 17 19:44:15 2016 +0200 +++ b/frontends/src/quick_frontend/quick_chat.py Mon Jul 18 00:52:02 2016 +0200 @@ -278,18 +278,18 @@ return True return False - def updateHistory(self, size=C.HISTORY_LIMIT_DEFAULT, search='', profile='@NONE@'): + def updateHistory(self, size=C.HISTORY_LIMIT_DEFAULT, filters=None, profile='@NONE@'): """Called when history need to be recreated Remove all message from history then call historyPrint Must probably be overriden by frontend to clear widget @param size (int): number of messages - @param search (str): pattern to filter the history results + @param filters (str): patterns to filter the history results @param profile (str): %(doc_profile)s """ self._locked = True self.messages.clear() - self.historyPrint(size, search, profile) + self.historyPrint(size, filters, profile) def _onHistoryPrinted(self): """Method called when history is printed (or failed) @@ -301,13 +301,15 @@ for data in self._cache: self.messageNew(*data) - def historyPrint(self, size=C.HISTORY_LIMIT_DEFAULT, search='', profile='@NONE@'): + def historyPrint(self, size=C.HISTORY_LIMIT_DEFAULT, filters=None, profile='@NONE@'): """Print the current history @param size (int): number of messages @param search (str): pattern to filter the history results @param profile (str): %(doc_profile)s """ + if filters is None: + filters = {} if size == 0: log.debug(u"Empty history requested, skipping") self._onHistoryPrinted() @@ -340,7 +342,7 @@ log.error(_(u"Can't get history")) self._onHistoryPrinted() - self.host.bridge.historyGet(unicode(self.host.profiles[profile].whoami.bare), unicode(target), size, True, search, profile, callback=_historyGetCb, errback=_historyGetEb) + self.host.bridge.historyGet(unicode(self.host.profiles[profile].whoami.bare), unicode(target), size, True, filters, profile, callback=_historyGetCb, errback=_historyGetEb) def messageNew(self, uid, timestamp, from_jid, to_jid, msg, subject, type_, extra, profile): log.debug(u"messageNew ==> {}".format((uid, timestamp, from_jid, to_jid, msg, subject, type_, extra, profile))) diff -r 53587e738aca -r b536dd121da1 src/bridge/DBus.py --- a/src/bridge/DBus.py Sun Jul 17 19:44:15 2016 +0200 +++ b/src/bridge/DBus.py Mon Jul 18 00:52:02 2016 +0200 @@ -399,10 +399,10 @@ return self._callback("getWaitingSub", unicode(profile_key)) @dbus.service.method(const_INT_PREFIX+const_CORE_SUFFIX, - in_signature='ssibss', out_signature='a(sdssa{ss}a{ss}sa{ss})', + in_signature='ssiba{ss}s', out_signature='a(sdssa{ss}a{ss}sa{ss})', async_callbacks=('callback', 'errback')) - def historyGet(self, from_jid, to_jid, limit, between=True, search='', profile="@NONE@", callback=None, errback=None): - return self._callback("historyGet", unicode(from_jid), unicode(to_jid), limit, between, unicode(search), unicode(profile), callback=callback, errback=errback) + def historyGet(self, from_jid, to_jid, limit, between=True, filters='', profile="@NONE@", callback=None, errback=None): + return self._callback("historyGet", unicode(from_jid), unicode(to_jid), limit, between, filters, unicode(profile), callback=callback, errback=errback) @dbus.service.method(const_INT_PREFIX+const_CORE_SUFFIX, in_signature='s', out_signature='b', diff -r 53587e738aca -r b536dd121da1 src/bridge/bridge_constructor/bridge_template.ini --- a/src/bridge/bridge_constructor/bridge_template.ini Sun Jul 17 19:44:15 2016 +0200 +++ b/src/bridge/bridge_constructor/bridge_template.ini Mon Jul 18 00:52:02 2016 +0200 @@ -591,7 +591,7 @@ async= type=method category=core -sig_in=ssibss +sig_in=ssiba{ss}s sig_out=a(sdssa{ss}a{ss}sa{ss}) param_3_default=True param_4_default='' @@ -601,7 +601,11 @@ doc_param_1=to_jid: dest JID (bare jid for catch all, full jid else) doc_param_2=limit: max number of history elements to get (0 for the whole history) doc_param_3=between: True if we want history between the two jids (in both direction), False if we only want messages from from_jid to to_jid -doc_param_4=search: pattern to filter the history results +doc_param_4=filters: patterns to filter the history results, can be: + - body: pattern must be in message body + - search: pattern must be in message body or source resource + - types: type must one of those, values are separated by spaces + - not_types: type must not be one of those, values are separated by spaces doc_param_5=%(doc_profile)s doc_return=Ordered list (by timestamp) of data as in [messageNew] (without final profile) diff -r 53587e738aca -r b536dd121da1 src/memory/memory.py --- a/src/memory/memory.py Sun Jul 17 19:44:15 2016 +0200 +++ b/src/memory/memory.py Mon Jul 18 00:52:02 2016 +0200 @@ -513,10 +513,10 @@ def addToHistory(self, client, data): return self.storage.addToHistory(data, client.profile) - def _historyGet(self, from_jid_s, to_jid_s, limit=C.HISTORY_LIMIT_NONE, between=True, search=None, profile=C.PROF_KEY_NONE): - return self.historyGet(jid.JID(from_jid_s), jid.JID(to_jid_s), limit, between, search, profile) + def _historyGet(self, from_jid_s, to_jid_s, limit=C.HISTORY_LIMIT_NONE, between=True, filters=None, profile=C.PROF_KEY_NONE): + return self.historyGet(jid.JID(from_jid_s), jid.JID(to_jid_s), limit, between, filters, profile) - def historyGet(self, from_jid, to_jid, limit=C.HISTORY_LIMIT_NONE, between=True, search=None, profile=C.PROF_KEY_NONE): + def historyGet(self, from_jid, to_jid, limit=C.HISTORY_LIMIT_NONE, between=True, filters=None, profile=C.PROF_KEY_NONE): """Retrieve messages in history @param from_jid (JID): source JID (full, or bare for catchall) @@ -526,7 +526,7 @@ - C.HISTORY_LIMIT_NONE or None for unlimited - C.HISTORY_LIMIT_DEFAULT to use the HISTORY_LIMIT parameter value @param between (bool): confound source and dest (ignore the direction) - @param search (str): pattern to filter the history results + @param filters (str): pattern to filter the history results (see bridge API for details) @param profile (str): %(doc_profile)s @return (D(list)): list of message data as in [messageNew] """ @@ -537,7 +537,7 @@ limit = None if limit == 0: return defer.succeed([]) - return self.storage.historyGet(from_jid, to_jid, limit, between, search, profile) + return self.storage.historyGet(from_jid, to_jid, limit, between, filters, profile) ## Statuses ## diff -r 53587e738aca -r b536dd121da1 src/memory/sqlite.py --- a/src/memory/sqlite.py Sun Jul 17 19:44:15 2016 +0200 +++ b/src/memory/sqlite.py Mon Jul 18 00:52:02 2016 +0200 @@ -392,7 +392,7 @@ ret.append((m['uid'], m['timestamp'], m['from'], m['to'], m['message'], m['subject'], m['type'], m['extra'])) return ret - def historyGet(self, from_jid, to_jid, limit=None, between=True, search=None, profile=None): + def historyGet(self, from_jid, to_jid, limit=None, between=True, filters=None, profile=None): """Retrieve messages in history @param from_jid (JID): source JID (full, or bare for catchall) @@ -406,10 +406,12 @@ @return: list of tuple as in [messageNew] """ assert profile + if filters is None: + filters = {} if limit == 0: return defer.succeed([]) - query_parts = ["SELECT uid, update_uid, source, dest, source_res, dest_res, timestamp, received_timestamp,\ + query_parts = [u"SELECT uid, update_uid, source, dest, source_res, dest_res, timestamp, received_timestamp,\ type, extra, message, message.language, subject, subject.language, thread_id, thread.parent_id\ FROM history LEFT JOIN message ON history.uid = message.history_uid\ LEFT JOIN subject ON history.uid=subject.history_uid\ @@ -421,29 +423,43 @@ values.append(_jid.userhost()) if _jid.resource: values.append(_jid.resource) - return '(%s=? AND %s_res=?)' % (type_, type_) - return '%s=?' % (type_, ) + return u'(%s=? AND %s_res=?)' % (type_, type_) + return u'%s=?' % (type_, ) if between: - query_parts.append("((%s AND %s) OR (%s AND %s))" % (test_jid('source', from_jid), + query_parts.append(u"((%s AND %s) OR (%s AND %s))" % (test_jid('source', from_jid), test_jid('dest', to_jid), test_jid('source', to_jid), test_jid('dest', from_jid))) else: - query_parts.append("%s AND %s" % (test_jid('source', from_jid), + query_parts.append(u"%s AND %s" % (test_jid('source', from_jid), test_jid('dest', to_jid))) - if search: - # TODO: use REGEXP (function to be defined) instead of GLOB: https://www.sqlite.org/lang_expr.html - query_parts.append("AND message GLOB ?") - values.append("*%s*" % search) - query_parts.append("ORDER BY timestamp DESC") # we reverse the order in sqliteHistoryToList + if filters: + if 'body' in filters: + # TODO: use REGEXP (function to be defined) instead of GLOB: https://www.sqlite.org/lang_expr.html + query_parts.append(u"AND message LIKE ?") + values.append(u"%{}%".format(filters['body'])) + if 'search' in filters: + query_parts.append(u"AND (message LIKE ? OR source_res LIKE ?)") + values.extend([u"%{}%".format(filters['search'])] * 2) + if 'types' in filters: + types = filters['types'].split() + query_parts.append(u"AND type IN ({})".format(u','.join("?"*len(types)))) + values.extend(types) + if 'not_types' in filters: + types = filters['not_types'].split() + query_parts.append(u"AND type NOT IN ({})".format(u','.join("?"*len(types)))) + values.extend(types) + + + query_parts.append(u"ORDER BY timestamp DESC") # we reverse the order in sqliteHistoryToList # we use DESC here so LIMIT keep the last messages if limit is not None: - query_parts.append("LIMIT ?") + query_parts.append(u"LIMIT ?") values.append(limit) - d = self.dbpool.runQuery(" ".join(query_parts), values) + d = self.dbpool.runQuery(u" ".join(query_parts), values) d.addCallback(self.sqliteHistoryToList) d.addCallback(self.listDict2listTuple) return d