changeset 3709:09f5ac48ffe3

merge bookmark @
author Goffi <goffi@goffi.org>
date Fri, 12 Nov 2021 17:21:24 +0100 (2021-11-12)
parents 8353cc3b8db9 (current diff) 09112b1c3e05 (diff)
children b9718216a1c0
files docker/backend-dev-e2e/Dockerfile sat/core/xmpp.py sat/plugins/plugin_comp_file_sharing.py sat/plugins/plugin_exp_events.py sat/plugins/plugin_misc_text_syntaxes.py sat/plugins/plugin_xep_0384.py sat_frontends/jp/base.py setup.py
diffstat 28 files changed, 326 insertions(+), 200 deletions(-) [+]
line wrap: on
line diff
--- a/.hgignore	Mon Sep 27 08:29:09 2021 +0200
+++ b/.hgignore	Fri Nov 12 17:21:24 2021 +0100
@@ -7,6 +7,7 @@
 twistd.log
 twistd.pid
 bridge_constructor/generated
+tests/e2e/report*
 _trial_temp/
 sat.egg-info
 *.un~
--- a/CHANGELOG	Mon Sep 27 08:29:09 2021 +0200
+++ b/CHANGELOG	Fri Nov 12 17:21:24 2021 +0100
@@ -100,7 +100,7 @@
         - bulma theme becomes the default one for default site
         - "tickets" have been renamed to more generic term "lists"
         - (lists) common lists can be created from templates
-        - (photos) album can now be created or deleted from Libervia
+        - (photos) album can now be created or deleted from Libervia Web
         - (photos/album) photos or videos can now be uploaded from Libervia
         - (photos/album) a mobile friendly slideshow can be run
         - (photos/album) affiliations can be modified with new invitations manager
--- a/docker/backend-dev-demo/Dockerfile	Mon Sep 27 08:29:09 2021 +0200
+++ b/docker/backend-dev-demo/Dockerfile	Fri Nov 12 17:21:24 2021 +0100
@@ -16,7 +16,7 @@
 
 RUN ./entrypoint.sh \
     # we create the file sharing component which will autoconnect when backend is started
-    jp profile create file_sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file_sharing -A && \
+    jp profile create file-sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file-sharing -A && \
     libervia-backend stop
 
 RUN ./entrypoint.sh \
--- a/docker/backend-dev-demo/libervia.conf	Mon Sep 27 08:29:09 2021 +0200
+++ b/docker/backend-dev-demo/libervia.conf	Fri Nov 12 17:21:24 2021 +0100
@@ -4,7 +4,7 @@
 email_port = 8025
 email_admins_list = admin@server1.test
 
-[component file_sharing]
+[component file-sharing]
 http_upload_public_facing_url = http://localhost:7777
 http_upload_connection_type = http
 http_upload_port = 7777
--- a/docker/backend-dev-e2e/Dockerfile	Mon Sep 27 08:29:09 2021 +0200
+++ b/docker/backend-dev-e2e/Dockerfile	Fri Nov 12 17:21:24 2021 +0100
@@ -1,5 +1,5 @@
-ARG REVISION=dev
-FROM libervia/backend:${REVISION}
+ARG REVISION
+FROM libervia/backend:${REVISION:-dev}
 
 LABEL maintainer="Goffi <tmp_dockerfiles@goffi.org>"
 
@@ -58,7 +58,7 @@
 
 RUN ./entrypoint.sh \
     # we create the file sharing component which will autoconnect when backend is started
-    li profile create file_sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file_sharing -A && \
+    li profile create file-sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file-sharing -A && \
     libervia-backend stop
 
 ENV LIBERVIA_TEST_REPORT_DIR=/reports
--- a/docker/backend-dev-e2e/libervia.conf	Mon Sep 27 08:29:09 2021 +0200
+++ b/docker/backend-dev-e2e/libervia.conf	Fri Nov 12 17:21:24 2021 +0100
@@ -4,7 +4,7 @@
 email_port = 8025
 email_admins_list = admin@server1.test
 
-[component file_sharing]
+[component file-sharing]
 tls_certificate = /usr/share/libervia/certificates/server1.test.pem
 tls_private_key = /usr/share/libervia/certificates/server1.test-key.pem
 http_upload_public_facing_url = https://libervia-backend.test:8888
--- a/docker/backend-dev/Dockerfile	Mon Sep 27 08:29:09 2021 +0200
+++ b/docker/backend-dev/Dockerfile	Fri Nov 12 17:21:24 2021 +0100
@@ -36,12 +36,12 @@
     # local repos without it them cause troubles
     hg clone https://repos.goffi.org/urwid-satext && ~/libervia_env/bin/pip install -e urwid-satext && \
     mv urwid-satext/urwid_satext.egg-info ~/libervia_env/lib/python3.*/site-packages && \
-    hg clone https://repos.goffi.org/sat_tmp -u "${REVISION:-tip}" && ~/libervia_env/bin/pip install -e sat_tmp && \
+    hg clone https://repos.goffi.org/sat_tmp -u "${REVISION:-@}" && ~/libervia_env/bin/pip install -e sat_tmp && \
     mv sat_tmp/sat_tmp.egg-info ~/libervia_env/lib/python3.*/site-packages && \
     hg clone https://repos.goffi.org/libervia-templates && ~/libervia_env/bin/pip install -e libervia-templates && \
     mv libervia-templates/libervia_templates.egg-info ~/libervia_env/lib/python3.*/site-packages && \
     hg clone https://repos.goffi.org/libervia-media && \
-    hg clone https://repos.goffi.org/libervia-backend -u "${REVISION:-tip}" && ~/libervia_env/bin/pip install -e 'libervia-backend[SVG]' && \
+    hg clone https://repos.goffi.org/libervia-backend -u "${REVISION:-@}" && ~/libervia_env/bin/pip install -e 'libervia-backend[SVG]' && \
     mv libervia-backend/libervia_backend.egg-info ~/libervia_env/lib/python3.*/site-packages && \
     mkdir -p /home/libervia/.local/share/libervia
 
--- a/docker/docker-compose-e2e.yml	Mon Sep 27 08:29:09 2021 +0200
+++ b/docker/docker-compose-e2e.yml	Fri Nov 12 17:21:24 2021 +0100
@@ -7,7 +7,6 @@
     depends_on:
       # we need to depend on backend to get IP address of the container for conf
       - backend
-    tmpfs: /var/lib/prosody
     tmpfs: /var/log/prosody
     networks:
       default:
--- a/docker/libervia-web-dev-demo/Dockerfile	Mon Sep 27 08:29:09 2021 +0200
+++ b/docker/libervia-web-dev-demo/Dockerfile	Fri Nov 12 17:21:24 2021 +0100
@@ -1,5 +1,5 @@
-ARG REVISION=dev
-FROM libervia/web:${REVISION}
+ARG REVISION
+FROM libervia/web:${REVISION:-dev}
 
 LABEL maintainer="Goffi <tmp_dockerfiles@goffi.org>"
 
--- a/docker/libervia-web-dev-e2e/Dockerfile	Mon Sep 27 08:29:09 2021 +0200
+++ b/docker/libervia-web-dev-e2e/Dockerfile	Fri Nov 12 17:21:24 2021 +0100
@@ -1,5 +1,5 @@
-ARG REVISION=dev
-FROM libervia/web:${REVISION}
+ARG REVISION
+FROM libervia/web:${REVISION:-dev}
 
 LABEL maintainer="Goffi <tmp_dockerfiles@goffi.org>"
 
--- a/docker/libervia-web-dev/Dockerfile	Mon Sep 27 08:29:09 2021 +0200
+++ b/docker/libervia-web-dev/Dockerfile	Fri Nov 12 17:21:24 2021 +0100
@@ -11,7 +11,7 @@
 RUN apt-get install -y --no-install-recommends yarnpkg
 WORKDIR /home/libervia
 USER libervia
-RUN cd /src && hg clone https://repos.goffi.org/libervia-web -u "${REVISION:-tip}" && \
+RUN cd /src && hg clone https://repos.goffi.org/libervia-web -u "${REVISION:-@}" && \
     ~/libervia_env/bin/pip install -e libervia-web && \
     mv libervia-web/libervia_web.egg-info ~/libervia_env/lib/python3.*/site-packages
 
--- a/docker/prosody-e2e/Dockerfile	Mon Sep 27 08:29:09 2021 +0200
+++ b/docker/prosody-e2e/Dockerfile	Fri Nov 12 17:21:24 2021 +0100
@@ -16,7 +16,3 @@
 COPY --chown=root:tls-cert certificates/server1.test/cert.pem /usr/share/libervia/certificates/server1.test.pem
 COPY --chown=root:tls-cert certificates/server1.test/key.pem /usr/share/libervia/certificates/server1.test-key.pem
 
-# we add exec to handle properly signals, this is missing upstream
-# FIXME: to be removed when new images are generated with
-#        https://github.com/prosody/prosody-docker/pull/65
-RUN sed -i "s/^runuser -u prosody/exec \0/" /entrypoint.sh
--- a/docker/prosody-e2e/entrypoint.sh	Mon Sep 27 08:29:09 2021 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,15 +0,0 @@
-#!/bin/bash
-set -e
-
-usermod -u "$(stat -c %u /var/lib/prosody/.)" prosody
-
-if [[ "$1" != "prosody" ]]; then
-    exec prosodyctl "$@"
-    exit 0;
-fi
-
-if [ "$LOCAL" -a  "$PASSWORD" -a "$DOMAIN" ] ; then
-    prosodyctl register "$LOCAL" "$DOMAIN" "$PASSWORD"
-fi
-
-exec runuser -u prosody -- "$@"
--- a/docker/web-demo.yml	Mon Sep 27 08:29:09 2021 +0200
+++ b/docker/web-demo.yml	Fri Nov 12 17:21:24 2021 +0100
@@ -6,8 +6,8 @@
     depends_on:
       # we need to depend on backend to get IP address of the container for conf
       - backend
-    tmpfs: /var/lib/prosody
-    tmpfs: /var/log/prosody
+    tmpfs:
+      - /var/log/prosody
     networks:
       default:
         aliases:
--- a/requirements.txt	Mon Sep 27 08:29:09 2021 +0200
+++ b/requirements.txt	Fri Nov 12 17:21:24 2021 +0100
@@ -1,22 +1,18 @@
 attrs==21.2.0
 Automat==20.2.0
 Babel==2.9.1
-cairocffi==1.2.0
-CairoSVG==2.5.2
-certifi==2021.5.30
-cffi==1.14.5
-chardet==4.0.0
+certifi==2021.10.8
+cffi==1.15.0
+charset-normalizer==2.0.7
 constantly==15.1.0
-cryptography==3.4.7
-cssselect2==0.4.1
-dbus-python==1.2.16
-defusedxml==0.7.1
+cryptography==3.4.8
+dbus-python==1.2.18
 DoubleRatchet==0.7.0
 html2text==2020.1.16
 hyperlink==21.0.0
-idna==2.10
+idna==3.3
 incremental==21.3.0
-Jinja2==3.0.1
+Jinja2==3.0.2
 langid==1.1.6
 lxml==4.6.3
 Markdown==3.3.4
@@ -24,38 +20,36 @@
 miniupnpc==2.0.2
 mutagen==1.45.1
 netifaces==0.11.0
-numpy==1.20.3
+numpy==1.21.3
 OMEMO==0.12.0
 omemo-backend-signal==0.2.6
-Pillow==8.2.0
-progressbar2==3.53.1
-protobuf==3.17.3
+Pillow==8.4.0
+progressbar2==3.53.3
+protobuf==3.19.1
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
 pycairo==1.20.1
 pycparser==2.20
 pycrypto==2.6.1
-Pygments==2.9.0
-PyGObject==3.40.1
+Pygments==2.10.0
+PyGObject==3.40.0
 PyNaCl==1.4.0
 pyOpenSSL==20.0.1
-python-dateutil==2.8.1
+python-dateutil==2.8.2
 python-potr==1.0.2
 python-utils==2.5.6
-pytz==2021.1
+pytz==2021.3
 pyxdg==0.27
 PyYAML==5.4.1
-requests==2.25.1
+requests==2.26.0
 service-identity==21.1.0
-setuptools-scm==6.0.1
 shortuuid==1.0.1
 six==1.16.0
-tinycss2==1.1.0
 treq==21.5.0
 Twisted==21.2.0
-urllib3==1.26.5
+typing-extensions==3.10.0.2
+urllib3==1.26.7
 urwid==2.1.2
-webencodings==0.5.1
 wokkel==18.0.0
 X3DH==0.5.9
 XEdDSA==0.4.7
--- a/sat/core/xmpp.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/sat/core/xmpp.py	Fri Nov 12 17:21:24 2021 +0100
@@ -303,7 +303,11 @@
         self.streamInitialized()
 
     def _finish_connection(self, __):
-        self.conn_deferred.callback(None)
+        if self.conn_deferred.called:
+            # can happen in case of forced disconnection by server
+            log.debug(f"{self} has already been connected")
+        else:
+            self.conn_deferred.callback(None)
 
     def streamInitialized(self):
         """Called after _authd"""
--- a/sat/plugins/plugin_comp_file_sharing.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/sat/plugins/plugin_comp_file_sharing.py	Fri Nov 12 17:21:24 2021 +0100
@@ -545,7 +545,7 @@
         self._f.openFileWrite(
             client, file_tmp_path, transfer_data, file_data, stream_object
         )
-        return False, defer.succeed(True)
+        return False, True
 
     async def _retrieveFiles(
         self, client, session, content_data, content_name, file_data, file_elt
--- a/sat/plugins/plugin_exp_events.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/sat/plugins/plugin_exp_events.py	Fri Nov 12 17:21:24 2021 +0100
@@ -428,6 +428,7 @@
                 )
             else:
                 timestamp, data = self._parseEventElt(event_elt)
+                data["interest_id"] = item["id"]
                 events.append((timestamp, data))
         defer.returnValue(events)
 
--- a/sat/plugins/plugin_misc_email_invitation.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/sat/plugins/plugin_misc_email_invitation.py	Fri Nov 12 17:21:24 2021 +0100
@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import shortuuid
+from typing import Optional
 from twisted.internet import defer
 from twisted.words.protocols.jabber import jid
 from twisted.words.protocols.jabber import error
@@ -131,6 +132,97 @@
                 kwargs[key] = str(value)
         return defer.ensureDeferred(self.create(**kwargs))
 
+    async def getExistingInvitation(self, email: Optional[str]) -> Optional[dict]:
+        """Retrieve existing invitation with given email
+
+        @param email: check if any invitation exist with this email
+        @return: first found invitation, or None if nothing found
+        """
+        # FIXME: This method is highly inefficient, it get all invitations and check them
+        # one by one, this is just a temporary way to avoid creating creating new accounts
+        # for an existing email. A better way will be available with Libervia 0.9.
+        # TODO: use a better way to check existing invitations
+
+        if email is None:
+            return None
+        all_invitations = await self.invitations.all()
+        for id_, invitation in all_invitations.items():
+            if invitation.get("email") == email:
+                invitation[KEY_ID] = id_
+                return invitation
+
+    async def _createAccountAndProfile(
+        self,
+        id_: str,
+        kwargs: dict,
+        extra: dict
+    ) -> None:
+        """Create XMPP account and Libervia profile for guest"""
+        ## XMPP account creation
+        password = kwargs.pop('password', None)
+        if password is None:
+           password = utils.generatePassword()
+        assert password
+        # XXX: password is here saved in clear in database
+        #      it is needed for invitation as the same password is used for profile
+        #      and SàT need to be able to automatically open the profile with the uuid
+        # FIXME: we could add an extra encryption key which would be used with the
+        #        uuid when the invitee is connecting (e.g. with URL). This key would
+        #        not be saved and could be used to encrypt profile password.
+        extra[KEY_PASSWORD] = password
+
+        jid_ = kwargs.pop('jid_', None)
+        if not jid_:
+            domain = self.host.memory.getConfig(None, 'xmpp_domain')
+            if not domain:
+                # TODO: fallback to profile's domain
+                raise ValueError(_("You need to specify xmpp_domain in sat.conf"))
+            jid_ = "invitation-{uuid}@{domain}".format(uuid=shortuuid.uuid(),
+                                                        domain=domain)
+        jid_ = jid.JID(jid_)
+        extra[KEY_JID] = jid_.full()
+
+        if jid_.user:
+            # we don't register account if there is no user as anonymous login is then
+            # used
+            try:
+                await self.host.plugins['XEP-0077'].registerNewAccount(jid_, password)
+            except error.StanzaError as e:
+                prefix = jid_.user
+                idx = 0
+                while e.condition == 'conflict':
+                    if idx >= SUFFIX_MAX:
+                        raise exceptions.ConflictError(_("Can't create XMPP account"))
+                    jid_.user = prefix + '_' + str(idx)
+                    log.info(_("requested jid already exists, trying with {}".format(
+                        jid_.full())))
+                    try:
+                        await self.host.plugins['XEP-0077'].registerNewAccount(
+                            jid_,
+                            password
+                        )
+                    except error.StanzaError:
+                        idx += 1
+                    else:
+                        break
+                if e.condition != 'conflict':
+                    raise e
+
+            log.info(_("account {jid_} created").format(jid_=jid_.full()))
+
+        ## profile creation
+
+        extra[KEY_GUEST_PROFILE] = guest_profile = INVITEE_PROFILE_TPL.format(
+            uuid=id_
+        )
+        # profile creation should not fail as we generate unique name ourselves
+        await self.host.memory.createProfile(guest_profile, password)
+        await self.host.memory.startSession(password, guest_profile)
+        await self.host.memory.setParam("JabberID", jid_.full(), "Connection",
+                                        profile_key=guest_profile)
+        await self.host.memory.setParam("Password", password, "Connection",
+                                        profile_key=guest_profile)
+
     async def create(self, **kwargs):
         r"""Create an invitation
 
@@ -201,6 +293,13 @@
         self.checkExtra(extra)
 
         email = kwargs.pop('email', None)
+
+        existing = await self.getExistingInvitation(email)
+        if existing is not None:
+            log.info(f"There is already an invitation for {email!r}")
+            extra.update(existing)
+            del extra[KEY_ID]
+
         emails_extra = kwargs.pop('emails_extra', [])
         if not email and emails_extra:
             raise ValueError(
@@ -214,67 +313,18 @@
 
         ## uuid
         log.info(_("creating an invitation"))
-        id_ = str(shortuuid.uuid())
+        id_ = existing[KEY_ID] if existing else str(shortuuid.uuid())
 
-        ## XMPP account creation
-        password = kwargs.pop('password', None)
-        if password is None:
-           password = utils.generatePassword()
-        assert password
-        # XXX: password is here saved in clear in database
-        #      it is needed for invitation as the same password is used for profile
-        #      and SàT need to be able to automatically open the profile with the uuid
-        # FIXME: we could add an extra encryption key which would be used with the uuid
-        #        when the invitee is connecting (e.g. with URL). This key would not be
-        #        saved and could be used to encrypt profile password.
-        extra[KEY_PASSWORD] = password
+        if existing is None:
+            await self._createAccountAndProfile(id_, kwargs, extra)
 
-        jid_ = kwargs.pop('jid_', None)
-        if not jid_:
-            domain = self.host.memory.getConfig(None, 'xmpp_domain')
-            if not domain:
-                # TODO: fallback to profile's domain
-                raise ValueError(_("You need to specify xmpp_domain in sat.conf"))
-            jid_ = "invitation-{uuid}@{domain}".format(uuid=shortuuid.uuid(),
-                                                        domain=domain)
-        jid_ = jid.JID(jid_)
-        if jid_.user:
-            # we don't register account if there is no user as anonymous login is then
-            # used
-            try:
-                await self.host.plugins['XEP-0077'].registerNewAccount(jid_, password)
-            except error.StanzaError as e:
-                prefix = jid_.user
-                idx = 0
-                while e.condition == 'conflict':
-                    if idx >= SUFFIX_MAX:
-                        raise exceptions.ConflictError(_("Can't create XMPP account"))
-                    jid_.user = prefix + '_' + str(idx)
-                    log.info(_("requested jid already exists, trying with {}".format(
-                        jid_.full())))
-                    try:
-                        await self.host.plugins['XEP-0077'].registerNewAccount(jid_,
-                                                                               password)
-                    except error.StanzaError:
-                        idx += 1
-                    else:
-                        break
-                if e.condition != 'conflict':
-                    raise e
+        profile = kwargs.pop('profile', None)
+        guest_profile = extra[KEY_GUEST_PROFILE]
+        jid_ = jid.JID(extra[KEY_JID])
 
-            log.info(_("account {jid_} created").format(jid_=jid_.full()))
-
-        ## profile creation
-
-        extra[KEY_GUEST_PROFILE] = guest_profile = INVITEE_PROFILE_TPL.format(uuid=id_)
-        # profile creation should not fail as we generate unique name ourselves
-        await self.host.memory.createProfile(guest_profile, password)
-        await self.host.memory.startSession(password, guest_profile)
-        await self.host.memory.setParam("JabberID", jid_.full(), "Connection",
-                                        profile_key=guest_profile)
-        await self.host.memory.setParam("Password", password, "Connection",
-                                        profile_key=guest_profile)
+        ## identity
         name = kwargs.pop('name', None)
+        password = extra[KEY_PASSWORD]
         if name is not None:
             extra['name'] = name
             try:
@@ -306,7 +356,6 @@
             else:
                 format_args['name'] = name
 
-            profile = kwargs.pop('profile', None)
             if profile is None:
                 format_args['profile'] = ''
             else:
@@ -344,8 +393,6 @@
         if kwargs:
             log.warning(_("Not all arguments have been consumed: {}").format(kwargs))
 
-        extra[KEY_JID] = jid_.full()
-
         ## extra data saving
         self.invitations[id_] = extra
 
--- a/sat/plugins/plugin_misc_text_syntaxes.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/sat/plugins/plugin_misc_text_syntaxes.py	Fri Nov 12 17:21:24 2021 +0100
@@ -17,16 +17,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import re
+from functools import partial
 from html import escape
-from functools import partial
-from sat.core.i18n import _, D_
-from sat.core.constants import Const as C
-from sat.core.log import getLogger
+import re
+from typing import Set
 
 from twisted.internet import defer
 from twisted.internet.threads import deferToThread
+
 from sat.core import exceptions
+from sat.core.constants import Const as C
+from sat.core.i18n import D_, _
+from sat.core.log import getLogger
 from sat.tools import xml_tools
 
 try:
@@ -115,7 +117,15 @@
     "track",
     "wbr")
 
-SAFE_ATTRS = html.defs.safe_attrs.union(("style", "poster", "controls"))
+SAFE_ATTRS = html.defs.safe_attrs.union({"style", "poster", "controls"}) - {"id"}
+SAFE_CLASSES = {
+    # those classes are used for code highlighting
+    "bp", "c", "ch", "cm", "cp", "cpf", "cs", "dl", "err", "fm", "gd", "ge", "get", "gh",
+    "gi", "go", "gp", "gr", "gs", "gt", "gu", "highlight", "hll", "il", "k", "kc", "kd",
+    "kn", "kp", "kr", "kt", "m", "mb", "mf", "mh", "mi", "mo", "na", "nb", "nc", "nd",
+    "ne", "nf", "ni", "nl", "nn", "no", "nt", "nv", "o", "ow", "s", "sa", "sb", "sc",
+    "sd", "se", "sh", "si", "sr", "ss", "sx", "vc", "vg", "vi", "vm", "w", "write",
+}
 STYLES_VALUES_REGEX = (
     r"^("
     + "|".join(
@@ -237,7 +247,8 @@
         except ImportError:
             log.warning("markdown or html2text not found, can't use Markdown syntax")
             log.info(
-                "You can download/install them from https://pythonhosted.org/Markdown/ and https://github.com/Alir3z4/html2text/"
+                "You can download/install them from https://pythonhosted.org/Markdown/ "
+                "and https://github.com/Alir3z4/html2text/"
             )
         host.bridge.addMethod(
             "syntaxConvert",
@@ -288,14 +299,14 @@
         )
         return failure
 
-    def cleanStyle(self, styles):
+    def cleanStyle(self, styles_raw: str) -> str:
         """"Clean unsafe CSS styles
 
         Remove styles not in the whitelist, or where the value doesn't match the regex
-        @param styles_raw(unicode): CSS styles
-        @return (unicode): cleaned styles
+        @param styles_raw: CSS styles
+        @return: cleaned styles
         """
-        styles = styles.split(";")
+        styles: List[str] = styles_raw.split(";")
         cleaned_styles = []
         for style in styles:
             try:
@@ -315,6 +326,14 @@
             ["%s: %s" % (key_, value_) for key_, value_ in cleaned_styles]
         )
 
+    def cleanClasses(self, classes_raw: str) -> str:
+        """Remove any non whitelisted class
+
+        @param classes_raw: classes set on an element
+        @return: remaining classes (can be empty string)
+        """
+        return " ".join(SAFE_CLASSES.intersection(classes_raw.split()))
+
     def cleanXHTML(self, xhtml):
         """Clean XHTML text by removing potentially dangerous/malicious parts
 
@@ -341,6 +360,8 @@
         xhtml_elt = cleaner.clean_html(xhtml_elt)
         for elt in xhtml_elt.xpath("//*[@style]"):
             elt.set("style", self.cleanStyle(elt.get("style")))
+        for elt in xhtml_elt.xpath("//*[@class]"):
+            elt.set("class", self.cleanClasses(elt.get("class")))
         # we remove self-closing elements for non-void elements
         for element in xhtml_elt.iter(tag=etree.Element):
             if not element.text:
--- a/sat/plugins/plugin_xep_0054.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/sat/plugins/plugin_xep_0054.py	Fri Nov 12 17:21:24 2021 +0100
@@ -45,10 +45,7 @@
         "Missing module pillow, please download/install it from https://python-pillow.github.io"
     )
 
-try:
-    from twisted.words.protocols.xmlstream import XMPPHandler
-except ImportError:
-    from wokkel.subprotocols import XMPPHandler
+from twisted.words.protocols.jabber.xmlstream import XMPPHandler
 
 IMPORT_NAME = "XEP-0054"
 
@@ -109,7 +106,7 @@
             NS_VCARD, client.profile)
         await client._xep_0054_avatar_hashes.load()
 
-    def savePhoto(self, client, photo_elt, entity_jid):
+    def savePhoto(self, client, photo_elt, entity):
         """Parse a <PHOTO> photo_elt and save the picture"""
         # XXX: this method is launched in a separate thread
         try:
@@ -131,7 +128,7 @@
             raise Failure(exceptions.NotFound())
 
         if not buf:
-            log.warning("empty avatar for {jid}".format(jid=entity_jid.full()))
+            log.warning("empty avatar for {jid}".format(jid=entity.full()))
             raise Failure(exceptions.NotFound())
 
         log.debug(_("Decoding binary"))
@@ -140,7 +137,7 @@
 
         if mime_type is None:
             log.debug(
-                f"no media type found specified for {entity_jid}'s avatar, trying to "
+                f"no media type found specified for {entity}'s avatar, trying to "
                 f"guess")
 
             try:
@@ -149,7 +146,7 @@
                 log.warning(f"Can't open avatar buffer: {e}")
 
             if mime_type is None:
-                msg = f"Can't find media type for {entity_jid}'s avatar"
+                msg = f"Can't find media type for {entity}'s avatar"
                 log.warning(msg)
                 raise Failure(exceptions.DataError(msg))
 
@@ -435,19 +432,6 @@
     def getDiscoItems(self, requestor, target, nodeIdentifier=""):
         return []
 
-    def _checkAvatarHash(self, client, entity, given_hash):
-        """Check that hash in cache (i.e. computed hash) is the same as given one"""
-        # XXX: if they differ, the avatar will be requested on each connection
-        # TODO: try to avoid re-requesting avatar in this case
-        computed_hash = client._xep_0054_avatar_hashes[entity.full()]
-        if computed_hash != given_hash:
-            log.warning(
-                "computed hash differs from given hash for {entity}:\n"
-                "computed: {computed}\ngiven: {given}".format(
-                    entity=entity, computed=computed_hash, given=given_hash
-                )
-            )
-
     async def update(self, presence):
         """Called on <presence/> stanza with vcard data
 
@@ -468,19 +452,19 @@
         except StopIteration:
             return
 
-        new_hash = str(photo_elt).strip()
-        if new_hash == HASH_SHA1_EMPTY:
-            new_hash = ""
+        given_hash = str(photo_elt).strip()
+        if given_hash == HASH_SHA1_EMPTY:
+            given_hash = ""
 
         hashes_cache = client._xep_0054_avatar_hashes
 
         old_hash = hashes_cache.get(entity_jid.full())
 
-        if old_hash == new_hash:
+        if old_hash == given_hash:
             # no change, we can return…
-            if new_hash:
+            if given_hash:
                 # …but we double check that avatar is in cache
-                avatar_cache = self.host.common_cache.getMetadata(new_hash)
+                avatar_cache = self.host.common_cache.getMetadata(given_hash)
                 if avatar_cache is None:
                     log.debug(
                         f"Avatar for [{entity_jid}] is known but not in cache, we get "
@@ -492,19 +476,19 @@
                     log.debug(f"avatar for {entity_jid} is already in cache")
             return
 
-        if new_hash is None:
+        if given_hash is None:
             # XXX: we use empty string to indicate that there is no avatar
-            new_hash = ""
+            given_hash = ""
 
-        await hashes_cache.aset(entity_jid.full(), new_hash)
+        await hashes_cache.aset(entity_jid.full(), given_hash)
 
-        if not new_hash:
+        if not given_hash:
             await self.plugin_parent._i.update(
                 client, IMPORT_NAME, "avatar", None, entity_jid)
             # the avatar has been removed, no need to go further
             return
 
-        avatar_cache = self.host.common_cache.getMetadata(new_hash)
+        avatar_cache = self.host.common_cache.getMetadata(given_hash)
         if avatar_cache is not None:
             log.debug(
                 f"New avatar found for [{entity_jid}], it's already in cache, we use it"
@@ -516,7 +500,7 @@
                     'path': avatar_cache['path'],
                     'filename': avatar_cache['filename'],
                     'media_type': avatar_cache['mime_type'],
-                    'cache_uid': new_hash,
+                    'cache_uid': given_hash,
                 },
                 entity_jid
             )
@@ -528,7 +512,14 @@
             if vcard is None:
                 log.warning(f"Unexpected empty vCard for {entity_jid}")
                 return
-            await self._checkAvatarHash(client, entity_jid, new_hash)
+            computed_hash = client._xep_0054_avatar_hashes[entity_jid.full()]
+            if computed_hash != given_hash:
+                log.warning(
+                    "computed hash differs from given hash for {entity}:\n"
+                    "computed: {computed}\ngiven: {given}".format(
+                        entity=entity_jid, computed=computed_hash, given=given_hash
+                    )
+                )
 
     def _update(self, presence):
         defer.ensureDeferred(self.update(presence))
--- a/sat/plugins/plugin_xep_0264.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/sat/plugins/plugin_xep_0264.py	Fri Nov 12 17:21:24 2021 +0100
@@ -73,7 +73,10 @@
     SIZE_MEDIUM = (640, 640)
     SIZE_BIG = (1280, 1280)
     SIZE_FULL_SCREEN = (2560, 2560)
-    SIZES = (SIZE_SMALL, SIZE_MEDIUM, SIZE_FULL_SCREEN)
+    # FIXME: SIZE_FULL_SCREEN is currently discarded as the resulting files are too big
+    # for BoB
+    # TODO: use an other mechanism than BoB for bigger files
+    SIZES = (SIZE_SMALL, SIZE_MEDIUM, SIZE_BIG)
 
     def __init__(self, host):
         log.info(_("Plugin XEP_0264 initialization"))
--- a/sat/plugins/plugin_xep_0384.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/sat/plugins/plugin_xep_0384.py	Fri Nov 12 17:21:24 2021 +0100
@@ -1428,13 +1428,12 @@
         header_elt = encrypted_elt.addElement('header')
         header_elt['sid'] = str(encryption_data['sid'])
 
-        for to_jid in to_jids:
-            bare_jid_s = to_jid.userhost()
-
-            for rid, data in encryption_data['keys'][bare_jid_s].items():
+        for key_data in encryption_data['keys'].values():
+            for rid, data in key_data.items():
                 key_elt = header_elt.addElement(
                     'key',
-                    content=b64enc(data['data']))
+                    content=b64enc(data['data'])
+                )
                 key_elt['rid'] = str(rid)
                 if data['pre_key']:
                     key_elt['prekey'] = 'true'
--- a/sat_frontends/jp/base.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/sat_frontends/jp/base.py	Fri Nov 12 17:21:24 2021 +0100
@@ -121,33 +121,65 @@
         return config.getConfig(self.sat_conf, section, name, default=default)
 
     def guess_background(self):
-        if not sys.stdin.isatty() or not sys.stdout.isatty():
-            return 'dark'
-        stdin_fd = sys.stdin.fileno()
-        old_settings = termios.tcgetattr(stdin_fd)
+        # cf. https://unix.stackexchange.com/a/245568 (thanks!)
         try:
-            tty.setraw(sys.stdin.fileno())
-            # we request background color
-            sys.stdout.write("\033]11;?\a")
-            sys.stdout.flush()
-            expected = "\033]11;rgb:"
-            for c in expected:
-                ch = sys.stdin.read(1)
-                if ch != c:
-                    # background id is not supported, we default to "dark"
-                    # TODO: log something?
-                    return 'dark'
-            red, green, blue = [int(c, 16)/65535 for c in sys.stdin.read(14).split('/')]
-            # '\a' is the last character
-            sys.stdin.read(1)
-        finally:
-            termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_settings)
+            # for VTE based terminals
+            vte_version = int(os.getenv("VTE_VERSION", 0))
+        except ValueError:
+            vte_version = 0
+
+        color_fg_bg = os.getenv("COLORFGBG")
 
-        lum = utils.per_luminance(red, green, blue)
-        if lum <= 0.5:
-            return 'dark'
+        if ((sys.stdin.isatty() and sys.stdout.isatty()
+             and (
+                 # XTerm
+                 os.getenv("XTERM_VERSION")
+                 # Konsole
+                 or os.getenv("KONSOLE_VERSION")
+                 # All VTE based terminals
+                 or vte_version >= 3502
+             ))):
+            # ANSI escape sequence
+            stdin_fd = sys.stdin.fileno()
+            old_settings = termios.tcgetattr(stdin_fd)
+            try:
+                tty.setraw(sys.stdin.fileno())
+                # we request background color
+                sys.stdout.write("\033]11;?\a")
+                sys.stdout.flush()
+                expected = "\033]11;rgb:"
+                for c in expected:
+                    ch = sys.stdin.read(1)
+                    if ch != c:
+                        # background id is not supported, we default to "dark"
+                        # TODO: log something?
+                        return 'dark'
+                red, green, blue = [
+                    int(c, 16)/65535 for c in sys.stdin.read(14).split('/')
+                ]
+                # '\a' is the last character
+                sys.stdin.read(1)
+            finally:
+                termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_settings)
+
+            lum = utils.per_luminance(red, green, blue)
+            if lum <= 0.5:
+                return 'dark'
+            else:
+                return 'light'
+        elif color_fg_bg:
+            # no luck with ANSI escape sequence, we try COLORFGBG environment variable
+            try:
+                bg = int(color_fg_bg.split(";")[-1])
+            except ValueError:
+                return "dark"
+            if bg in list(range(7)) + [8]:
+                return "dark"
+            else:
+                return "light"
         else:
-            return 'light'
+            # no autodetection method found
+            return "dark"
 
     def set_color_theme(self):
         background = self.get_config('background', default='auto')
@@ -159,6 +191,7 @@
                 'your settings in libervia.conf').format(
                     background=repr(background)
                 ))
+        self.background = background
         if background == 'light':
             C.A_HEADER = A.FG_MAGENTA
             C.A_SUBHEADER = A.BOLD + A.FG_RED
--- a/sat_frontends/jp/cmd_debug.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/sat_frontends/jp/cmd_debug.py	Fri Nov 12 17:21:24 2021 +0100
@@ -199,6 +199,7 @@
         pass
 
     async def start(self):
+        print(f"background currently used: {A.BOLD}{self.host.background}{A.RESET}\n")
         for attr in dir(C):
             if not attr.startswith("A_"):
                 continue
--- a/setup.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/setup.py	Fri Nov 12 17:21:24 2021 +0100
@@ -41,7 +41,7 @@
     'pygments < 3',
     'pygobject < 3.40.1',
     'pyopenssl < 21.0.0',
-    'python-dateutil < 3',
+    'python-dateutil >= 2.8.1, < 3',
     'python-potr < 1.1',
     'pyxdg < 0.30',
     'sat_tmp == 0.9.*',
--- a/tests/e2e/libervia-web/test_libervia-web.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/tests/e2e/libervia-web/test_libervia-web.py	Fri Nov 12 17:21:24 2021 +0100
@@ -21,7 +21,7 @@
 import pytest
 from helium import (
     go_to, write, press, click, drag_file, find_all, wait_until, S, Text, Link, Button,
-    get_driver, ENTER
+    select, scroll_down, get_driver, ENTER
 )
 
 
@@ -178,3 +178,54 @@
         click("test album")
         wait_until(lambda: not S("#loading_screen").exists())
         assert len(find_all(S("img.is-photo-thumbnail"))) == 2
+
+
+class TestLists:
+    TEST_GENERIC_LIST_URL = (
+        "https://libervia-web.test:8443/lists/view/account1@server1.test/"
+        "fdp%2Fsubmitted%2Forg.salut-a-toi.tickets%3A0_test-generic-list"
+    )
+
+    @pytest.mark.dependency(name="create_generic_list")
+    def test_user_can_create_generic(self, log_in_account1):
+        go_to("https://libervia-web.test:8443/lists/view")
+        click("create a list")
+        tickets_btn = S("//span[text()='Tickets']")
+        click(tickets_btn)
+        write("test-generic-list", "name of the list")
+        click("create list")
+        wait_until(Text("Success").exists)
+
+
+    @pytest.mark.dependency(
+        name="create_generic_list_item", depends=["create_generic_list"]
+    )
+    def test_user_can_create_generic_list_item(self):
+        go_to(self.TEST_GENERIC_LIST_URL)
+        click("create")
+        write("test item", "title")
+        write("label 1, label 2", "labels")
+        select("type", "feature request")
+        write("this is a test body", "body")
+        scroll_down(600)
+        click("create list item")
+        wait_until(Text("Success").exists)
+        assert Link("create").exists()
+        assert Link("manage invitations").exists()
+        item = Link("test item")
+        assert item.exists()
+        labels_elt = item.web_element.find_element_by_class_name("xmlui_field__labels")
+        labels = [t.text for t in labels_elt.find_elements_by_tag_name("span")]
+        assert labels == ["label 1", "label 2"]
+
+    @pytest.mark.dependency(depends=["create_generic_list_item"])
+    def test_list_item_can_be_displayed(self):
+        go_to(self.TEST_GENERIC_LIST_URL)
+        item = Link("test item")
+        # FIXME: we can't click on the item created above with Selenium, looks like a
+        #        Selenium bug, to be checked
+        go_to(item.href)
+        item_title = Text("test item")
+        assert item_title.exists()
+        item_body = Text("this is a test body")
+        assert item_body.exists()
--- a/tests/e2e/run_e2e.py	Mon Sep 27 08:29:09 2021 +0200
+++ b/tests/e2e/run_e2e.py	Fri Nov 12 17:21:24 2021 +0100
@@ -226,7 +226,7 @@
         docker_compose.up("-d")
 
         p = docker_compose.exec(
-            "-T", "--workdir", "/src/libervia-backend/tests", "backend",
+            "--workdir", "/src/libervia-backend/tests", "backend",
             "pytest", "-o", "cache_dir=/tmp", *sys.argv[1:], color="yes",
             _in=sys.stdin, _out=live_out, _out_bufsize=0, _err=live_err, _err_bufsize=0,
             _bg=True