Mercurial > libervia-backend
changeset 3709:09f5ac48ffe3
merge bookmark @
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Fri, 12 Nov 2021 17:21:24 +0100 |
| parents | 8353cc3b8db9 (current diff) 09112b1c3e05 (diff) |
| children | b9718216a1c0 |
| files | docker/backend-dev-e2e/Dockerfile sat/core/xmpp.py sat/plugins/plugin_comp_file_sharing.py sat/plugins/plugin_exp_events.py sat/plugins/plugin_misc_text_syntaxes.py sat/plugins/plugin_xep_0384.py sat_frontends/jp/base.py setup.py |
| diffstat | 28 files changed, 326 insertions(+), 200 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgignore Mon Sep 27 08:29:09 2021 +0200 +++ b/.hgignore Fri Nov 12 17:21:24 2021 +0100 @@ -7,6 +7,7 @@ twistd.log twistd.pid bridge_constructor/generated +tests/e2e/report* _trial_temp/ sat.egg-info *.un~
--- a/CHANGELOG Mon Sep 27 08:29:09 2021 +0200 +++ b/CHANGELOG Fri Nov 12 17:21:24 2021 +0100 @@ -100,7 +100,7 @@ - bulma theme becomes the default one for default site - "tickets" have been renamed to more generic term "lists" - (lists) common lists can be created from templates - - (photos) album can now be created or deleted from Libervia + - (photos) album can now be created or deleted from Libervia Web - (photos/album) photos or videos can now be uploaded from Libervia - (photos/album) a mobile friendly slideshow can be run - (photos/album) affiliations can be modified with new invitations manager
--- a/docker/backend-dev-demo/Dockerfile Mon Sep 27 08:29:09 2021 +0200 +++ b/docker/backend-dev-demo/Dockerfile Fri Nov 12 17:21:24 2021 +0100 @@ -16,7 +16,7 @@ RUN ./entrypoint.sh \ # we create the file sharing component which will autoconnect when backend is started - jp profile create file_sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file_sharing -A && \ + jp profile create file-sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file-sharing -A && \ libervia-backend stop RUN ./entrypoint.sh \
--- a/docker/backend-dev-demo/libervia.conf Mon Sep 27 08:29:09 2021 +0200 +++ b/docker/backend-dev-demo/libervia.conf Fri Nov 12 17:21:24 2021 +0100 @@ -4,7 +4,7 @@ email_port = 8025 email_admins_list = admin@server1.test -[component file_sharing] +[component file-sharing] http_upload_public_facing_url = http://localhost:7777 http_upload_connection_type = http http_upload_port = 7777
--- a/docker/backend-dev-e2e/Dockerfile Mon Sep 27 08:29:09 2021 +0200 +++ b/docker/backend-dev-e2e/Dockerfile Fri Nov 12 17:21:24 2021 +0100 @@ -1,5 +1,5 @@ -ARG REVISION=dev -FROM libervia/backend:${REVISION} +ARG REVISION +FROM libervia/backend:${REVISION:-dev} LABEL maintainer="Goffi <tmp_dockerfiles@goffi.org>" @@ -58,7 +58,7 @@ RUN ./entrypoint.sh \ # we create the file sharing component which will autoconnect when backend is started - li profile create file_sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file_sharing -A && \ + li profile create file-sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file-sharing -A && \ libervia-backend stop ENV LIBERVIA_TEST_REPORT_DIR=/reports
--- a/docker/backend-dev-e2e/libervia.conf Mon Sep 27 08:29:09 2021 +0200 +++ b/docker/backend-dev-e2e/libervia.conf Fri Nov 12 17:21:24 2021 +0100 @@ -4,7 +4,7 @@ email_port = 8025 email_admins_list = admin@server1.test -[component file_sharing] +[component file-sharing] tls_certificate = /usr/share/libervia/certificates/server1.test.pem tls_private_key = /usr/share/libervia/certificates/server1.test-key.pem http_upload_public_facing_url = https://libervia-backend.test:8888
--- a/docker/backend-dev/Dockerfile Mon Sep 27 08:29:09 2021 +0200 +++ b/docker/backend-dev/Dockerfile Fri Nov 12 17:21:24 2021 +0100 @@ -36,12 +36,12 @@ # local repos without it them cause troubles hg clone https://repos.goffi.org/urwid-satext && ~/libervia_env/bin/pip install -e urwid-satext && \ mv urwid-satext/urwid_satext.egg-info ~/libervia_env/lib/python3.*/site-packages && \ - hg clone https://repos.goffi.org/sat_tmp -u "${REVISION:-tip}" && ~/libervia_env/bin/pip install -e sat_tmp && \ + hg clone https://repos.goffi.org/sat_tmp -u "${REVISION:-@}" && ~/libervia_env/bin/pip install -e sat_tmp && \ mv sat_tmp/sat_tmp.egg-info ~/libervia_env/lib/python3.*/site-packages && \ hg clone https://repos.goffi.org/libervia-templates && ~/libervia_env/bin/pip install -e libervia-templates && \ mv libervia-templates/libervia_templates.egg-info ~/libervia_env/lib/python3.*/site-packages && \ hg clone https://repos.goffi.org/libervia-media && \ - hg clone https://repos.goffi.org/libervia-backend -u "${REVISION:-tip}" && ~/libervia_env/bin/pip install -e 'libervia-backend[SVG]' && \ + hg clone https://repos.goffi.org/libervia-backend -u "${REVISION:-@}" && ~/libervia_env/bin/pip install -e 'libervia-backend[SVG]' && \ mv libervia-backend/libervia_backend.egg-info ~/libervia_env/lib/python3.*/site-packages && \ mkdir -p /home/libervia/.local/share/libervia
--- a/docker/docker-compose-e2e.yml Mon Sep 27 08:29:09 2021 +0200 +++ b/docker/docker-compose-e2e.yml Fri Nov 12 17:21:24 2021 +0100 @@ -7,7 +7,6 @@ depends_on: # we need to depend on backend to get IP address of the container for conf - backend - tmpfs: /var/lib/prosody tmpfs: /var/log/prosody networks: default:
--- a/docker/libervia-web-dev-demo/Dockerfile Mon Sep 27 08:29:09 2021 +0200 +++ b/docker/libervia-web-dev-demo/Dockerfile Fri Nov 12 17:21:24 2021 +0100 @@ -1,5 +1,5 @@ -ARG REVISION=dev -FROM libervia/web:${REVISION} +ARG REVISION +FROM libervia/web:${REVISION:-dev} LABEL maintainer="Goffi <tmp_dockerfiles@goffi.org>"
--- a/docker/libervia-web-dev-e2e/Dockerfile Mon Sep 27 08:29:09 2021 +0200 +++ b/docker/libervia-web-dev-e2e/Dockerfile Fri Nov 12 17:21:24 2021 +0100 @@ -1,5 +1,5 @@ -ARG REVISION=dev -FROM libervia/web:${REVISION} +ARG REVISION +FROM libervia/web:${REVISION:-dev} LABEL maintainer="Goffi <tmp_dockerfiles@goffi.org>"
--- a/docker/libervia-web-dev/Dockerfile Mon Sep 27 08:29:09 2021 +0200 +++ b/docker/libervia-web-dev/Dockerfile Fri Nov 12 17:21:24 2021 +0100 @@ -11,7 +11,7 @@ RUN apt-get install -y --no-install-recommends yarnpkg WORKDIR /home/libervia USER libervia -RUN cd /src && hg clone https://repos.goffi.org/libervia-web -u "${REVISION:-tip}" && \ +RUN cd /src && hg clone https://repos.goffi.org/libervia-web -u "${REVISION:-@}" && \ ~/libervia_env/bin/pip install -e libervia-web && \ mv libervia-web/libervia_web.egg-info ~/libervia_env/lib/python3.*/site-packages
--- a/docker/prosody-e2e/Dockerfile Mon Sep 27 08:29:09 2021 +0200 +++ b/docker/prosody-e2e/Dockerfile Fri Nov 12 17:21:24 2021 +0100 @@ -16,7 +16,3 @@ COPY --chown=root:tls-cert certificates/server1.test/cert.pem /usr/share/libervia/certificates/server1.test.pem COPY --chown=root:tls-cert certificates/server1.test/key.pem /usr/share/libervia/certificates/server1.test-key.pem -# we add exec to handle properly signals, this is missing upstream -# FIXME: to be removed when new images are generated with -# https://github.com/prosody/prosody-docker/pull/65 -RUN sed -i "s/^runuser -u prosody/exec \0/" /entrypoint.sh
--- a/docker/prosody-e2e/entrypoint.sh Mon Sep 27 08:29:09 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,15 +0,0 @@ -#!/bin/bash -set -e - -usermod -u "$(stat -c %u /var/lib/prosody/.)" prosody - -if [[ "$1" != "prosody" ]]; then - exec prosodyctl "$@" - exit 0; -fi - -if [ "$LOCAL" -a "$PASSWORD" -a "$DOMAIN" ] ; then - prosodyctl register "$LOCAL" "$DOMAIN" "$PASSWORD" -fi - -exec runuser -u prosody -- "$@"
--- a/docker/web-demo.yml Mon Sep 27 08:29:09 2021 +0200 +++ b/docker/web-demo.yml Fri Nov 12 17:21:24 2021 +0100 @@ -6,8 +6,8 @@ depends_on: # we need to depend on backend to get IP address of the container for conf - backend - tmpfs: /var/lib/prosody - tmpfs: /var/log/prosody + tmpfs: + - /var/log/prosody networks: default: aliases:
--- a/requirements.txt Mon Sep 27 08:29:09 2021 +0200 +++ b/requirements.txt Fri Nov 12 17:21:24 2021 +0100 @@ -1,22 +1,18 @@ attrs==21.2.0 Automat==20.2.0 Babel==2.9.1 -cairocffi==1.2.0 -CairoSVG==2.5.2 -certifi==2021.5.30 -cffi==1.14.5 -chardet==4.0.0 +certifi==2021.10.8 +cffi==1.15.0 +charset-normalizer==2.0.7 constantly==15.1.0 -cryptography==3.4.7 -cssselect2==0.4.1 -dbus-python==1.2.16 -defusedxml==0.7.1 +cryptography==3.4.8 +dbus-python==1.2.18 DoubleRatchet==0.7.0 html2text==2020.1.16 hyperlink==21.0.0 -idna==2.10 +idna==3.3 incremental==21.3.0 -Jinja2==3.0.1 +Jinja2==3.0.2 langid==1.1.6 lxml==4.6.3 Markdown==3.3.4 @@ -24,38 +20,36 @@ miniupnpc==2.0.2 mutagen==1.45.1 netifaces==0.11.0 -numpy==1.20.3 +numpy==1.21.3 OMEMO==0.12.0 omemo-backend-signal==0.2.6 -Pillow==8.2.0 -progressbar2==3.53.1 -protobuf==3.17.3 +Pillow==8.4.0 +progressbar2==3.53.3 +protobuf==3.19.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycairo==1.20.1 pycparser==2.20 pycrypto==2.6.1 -Pygments==2.9.0 -PyGObject==3.40.1 +Pygments==2.10.0 +PyGObject==3.40.0 PyNaCl==1.4.0 pyOpenSSL==20.0.1 -python-dateutil==2.8.1 +python-dateutil==2.8.2 python-potr==1.0.2 python-utils==2.5.6 -pytz==2021.1 +pytz==2021.3 pyxdg==0.27 PyYAML==5.4.1 -requests==2.25.1 +requests==2.26.0 service-identity==21.1.0 -setuptools-scm==6.0.1 shortuuid==1.0.1 six==1.16.0 -tinycss2==1.1.0 treq==21.5.0 Twisted==21.2.0 -urllib3==1.26.5 +typing-extensions==3.10.0.2 +urllib3==1.26.7 urwid==2.1.2 -webencodings==0.5.1 wokkel==18.0.0 X3DH==0.5.9 XEdDSA==0.4.7
--- a/sat/core/xmpp.py Mon Sep 27 08:29:09 2021 +0200 +++ b/sat/core/xmpp.py Fri Nov 12 17:21:24 2021 +0100 @@ -303,7 +303,11 @@ self.streamInitialized() def _finish_connection(self, __): - self.conn_deferred.callback(None) + if self.conn_deferred.called: + # can happen in case of forced disconnection by server + log.debug(f"{self} has already been connected") + else: + self.conn_deferred.callback(None) def streamInitialized(self): """Called after _authd"""
--- a/sat/plugins/plugin_comp_file_sharing.py Mon Sep 27 08:29:09 2021 +0200 +++ b/sat/plugins/plugin_comp_file_sharing.py Fri Nov 12 17:21:24 2021 +0100 @@ -545,7 +545,7 @@ self._f.openFileWrite( client, file_tmp_path, transfer_data, file_data, stream_object ) - return False, defer.succeed(True) + return False, True async def _retrieveFiles( self, client, session, content_data, content_name, file_data, file_elt
--- a/sat/plugins/plugin_exp_events.py Mon Sep 27 08:29:09 2021 +0200 +++ b/sat/plugins/plugin_exp_events.py Fri Nov 12 17:21:24 2021 +0100 @@ -428,6 +428,7 @@ ) else: timestamp, data = self._parseEventElt(event_elt) + data["interest_id"] = item["id"] events.append((timestamp, data)) defer.returnValue(events)
--- a/sat/plugins/plugin_misc_email_invitation.py Mon Sep 27 08:29:09 2021 +0200 +++ b/sat/plugins/plugin_misc_email_invitation.py Fri Nov 12 17:21:24 2021 +0100 @@ -17,6 +17,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import shortuuid +from typing import Optional from twisted.internet import defer from twisted.words.protocols.jabber import jid from twisted.words.protocols.jabber import error @@ -131,6 +132,97 @@ kwargs[key] = str(value) return defer.ensureDeferred(self.create(**kwargs)) + async def getExistingInvitation(self, email: Optional[str]) -> Optional[dict]: + """Retrieve existing invitation with given email + + @param email: check if any invitation exist with this email + @return: first found invitation, or None if nothing found + """ + # FIXME: This method is highly inefficient, it get all invitations and check them + # one by one, this is just a temporary way to avoid creating creating new accounts + # for an existing email. A better way will be available with Libervia 0.9. + # TODO: use a better way to check existing invitations + + if email is None: + return None + all_invitations = await self.invitations.all() + for id_, invitation in all_invitations.items(): + if invitation.get("email") == email: + invitation[KEY_ID] = id_ + return invitation + + async def _createAccountAndProfile( + self, + id_: str, + kwargs: dict, + extra: dict + ) -> None: + """Create XMPP account and Libervia profile for guest""" + ## XMPP account creation + password = kwargs.pop('password', None) + if password is None: + password = utils.generatePassword() + assert password + # XXX: password is here saved in clear in database + # it is needed for invitation as the same password is used for profile + # and SàT need to be able to automatically open the profile with the uuid + # FIXME: we could add an extra encryption key which would be used with the + # uuid when the invitee is connecting (e.g. with URL). This key would + # not be saved and could be used to encrypt profile password. + extra[KEY_PASSWORD] = password + + jid_ = kwargs.pop('jid_', None) + if not jid_: + domain = self.host.memory.getConfig(None, 'xmpp_domain') + if not domain: + # TODO: fallback to profile's domain + raise ValueError(_("You need to specify xmpp_domain in sat.conf")) + jid_ = "invitation-{uuid}@{domain}".format(uuid=shortuuid.uuid(), + domain=domain) + jid_ = jid.JID(jid_) + extra[KEY_JID] = jid_.full() + + if jid_.user: + # we don't register account if there is no user as anonymous login is then + # used + try: + await self.host.plugins['XEP-0077'].registerNewAccount(jid_, password) + except error.StanzaError as e: + prefix = jid_.user + idx = 0 + while e.condition == 'conflict': + if idx >= SUFFIX_MAX: + raise exceptions.ConflictError(_("Can't create XMPP account")) + jid_.user = prefix + '_' + str(idx) + log.info(_("requested jid already exists, trying with {}".format( + jid_.full()))) + try: + await self.host.plugins['XEP-0077'].registerNewAccount( + jid_, + password + ) + except error.StanzaError: + idx += 1 + else: + break + if e.condition != 'conflict': + raise e + + log.info(_("account {jid_} created").format(jid_=jid_.full())) + + ## profile creation + + extra[KEY_GUEST_PROFILE] = guest_profile = INVITEE_PROFILE_TPL.format( + uuid=id_ + ) + # profile creation should not fail as we generate unique name ourselves + await self.host.memory.createProfile(guest_profile, password) + await self.host.memory.startSession(password, guest_profile) + await self.host.memory.setParam("JabberID", jid_.full(), "Connection", + profile_key=guest_profile) + await self.host.memory.setParam("Password", password, "Connection", + profile_key=guest_profile) + async def create(self, **kwargs): r"""Create an invitation @@ -201,6 +293,13 @@ self.checkExtra(extra) email = kwargs.pop('email', None) + + existing = await self.getExistingInvitation(email) + if existing is not None: + log.info(f"There is already an invitation for {email!r}") + extra.update(existing) + del extra[KEY_ID] + emails_extra = kwargs.pop('emails_extra', []) if not email and emails_extra: raise ValueError( @@ -214,67 +313,18 @@ ## uuid log.info(_("creating an invitation")) - id_ = str(shortuuid.uuid()) + id_ = existing[KEY_ID] if existing else str(shortuuid.uuid()) - ## XMPP account creation - password = kwargs.pop('password', None) - if password is None: - password = utils.generatePassword() - assert password - # XXX: password is here saved in clear in database - # it is needed for invitation as the same password is used for profile - # and SàT need to be able to automatically open the profile with the uuid - # FIXME: we could add an extra encryption key which would be used with the uuid - # when the invitee is connecting (e.g. with URL). This key would not be - # saved and could be used to encrypt profile password. - extra[KEY_PASSWORD] = password + if existing is None: + await self._createAccountAndProfile(id_, kwargs, extra) - jid_ = kwargs.pop('jid_', None) - if not jid_: - domain = self.host.memory.getConfig(None, 'xmpp_domain') - if not domain: - # TODO: fallback to profile's domain - raise ValueError(_("You need to specify xmpp_domain in sat.conf")) - jid_ = "invitation-{uuid}@{domain}".format(uuid=shortuuid.uuid(), - domain=domain) - jid_ = jid.JID(jid_) - if jid_.user: - # we don't register account if there is no user as anonymous login is then - # used - try: - await self.host.plugins['XEP-0077'].registerNewAccount(jid_, password) - except error.StanzaError as e: - prefix = jid_.user - idx = 0 - while e.condition == 'conflict': - if idx >= SUFFIX_MAX: - raise exceptions.ConflictError(_("Can't create XMPP account")) - jid_.user = prefix + '_' + str(idx) - log.info(_("requested jid already exists, trying with {}".format( - jid_.full()))) - try: - await self.host.plugins['XEP-0077'].registerNewAccount(jid_, - password) - except error.StanzaError: - idx += 1 - else: - break - if e.condition != 'conflict': - raise e + profile = kwargs.pop('profile', None) + guest_profile = extra[KEY_GUEST_PROFILE] + jid_ = jid.JID(extra[KEY_JID]) - log.info(_("account {jid_} created").format(jid_=jid_.full())) - - ## profile creation - - extra[KEY_GUEST_PROFILE] = guest_profile = INVITEE_PROFILE_TPL.format(uuid=id_) - # profile creation should not fail as we generate unique name ourselves - await self.host.memory.createProfile(guest_profile, password) - await self.host.memory.startSession(password, guest_profile) - await self.host.memory.setParam("JabberID", jid_.full(), "Connection", - profile_key=guest_profile) - await self.host.memory.setParam("Password", password, "Connection", - profile_key=guest_profile) + ## identity name = kwargs.pop('name', None) + password = extra[KEY_PASSWORD] if name is not None: extra['name'] = name try: @@ -306,7 +356,6 @@ else: format_args['name'] = name - profile = kwargs.pop('profile', None) if profile is None: format_args['profile'] = '' else: @@ -344,8 +393,6 @@ if kwargs: log.warning(_("Not all arguments have been consumed: {}").format(kwargs)) - extra[KEY_JID] = jid_.full() - ## extra data saving self.invitations[id_] = extra
--- a/sat/plugins/plugin_misc_text_syntaxes.py Mon Sep 27 08:29:09 2021 +0200 +++ b/sat/plugins/plugin_misc_text_syntaxes.py Fri Nov 12 17:21:24 2021 +0100 @@ -17,16 +17,18 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import re +from functools import partial from html import escape -from functools import partial -from sat.core.i18n import _, D_ -from sat.core.constants import Const as C -from sat.core.log import getLogger +import re +from typing import Set from twisted.internet import defer from twisted.internet.threads import deferToThread + from sat.core import exceptions +from sat.core.constants import Const as C +from sat.core.i18n import D_, _ +from sat.core.log import getLogger from sat.tools import xml_tools try: @@ -115,7 +117,15 @@ "track", "wbr") -SAFE_ATTRS = html.defs.safe_attrs.union(("style", "poster", "controls")) +SAFE_ATTRS = html.defs.safe_attrs.union({"style", "poster", "controls"}) - {"id"} +SAFE_CLASSES = { + # those classes are used for code highlighting + "bp", "c", "ch", "cm", "cp", "cpf", "cs", "dl", "err", "fm", "gd", "ge", "get", "gh", + "gi", "go", "gp", "gr", "gs", "gt", "gu", "highlight", "hll", "il", "k", "kc", "kd", + "kn", "kp", "kr", "kt", "m", "mb", "mf", "mh", "mi", "mo", "na", "nb", "nc", "nd", + "ne", "nf", "ni", "nl", "nn", "no", "nt", "nv", "o", "ow", "s", "sa", "sb", "sc", + "sd", "se", "sh", "si", "sr", "ss", "sx", "vc", "vg", "vi", "vm", "w", "write", +} STYLES_VALUES_REGEX = ( r"^(" + "|".join( @@ -237,7 +247,8 @@ except ImportError: log.warning("markdown or html2text not found, can't use Markdown syntax") log.info( - "You can download/install them from https://pythonhosted.org/Markdown/ and https://github.com/Alir3z4/html2text/" + "You can download/install them from https://pythonhosted.org/Markdown/ " + "and https://github.com/Alir3z4/html2text/" ) host.bridge.addMethod( "syntaxConvert", @@ -288,14 +299,14 @@ ) return failure - def cleanStyle(self, styles): + def cleanStyle(self, styles_raw: str) -> str: """"Clean unsafe CSS styles Remove styles not in the whitelist, or where the value doesn't match the regex - @param styles_raw(unicode): CSS styles - @return (unicode): cleaned styles + @param styles_raw: CSS styles + @return: cleaned styles """ - styles = styles.split(";") + styles: List[str] = styles_raw.split(";") cleaned_styles = [] for style in styles: try: @@ -315,6 +326,14 @@ ["%s: %s" % (key_, value_) for key_, value_ in cleaned_styles] ) + def cleanClasses(self, classes_raw: str) -> str: + """Remove any non whitelisted class + + @param classes_raw: classes set on an element + @return: remaining classes (can be empty string) + """ + return " ".join(SAFE_CLASSES.intersection(classes_raw.split())) + def cleanXHTML(self, xhtml): """Clean XHTML text by removing potentially dangerous/malicious parts @@ -341,6 +360,8 @@ xhtml_elt = cleaner.clean_html(xhtml_elt) for elt in xhtml_elt.xpath("//*[@style]"): elt.set("style", self.cleanStyle(elt.get("style"))) + for elt in xhtml_elt.xpath("//*[@class]"): + elt.set("class", self.cleanClasses(elt.get("class"))) # we remove self-closing elements for non-void elements for element in xhtml_elt.iter(tag=etree.Element): if not element.text:
--- a/sat/plugins/plugin_xep_0054.py Mon Sep 27 08:29:09 2021 +0200 +++ b/sat/plugins/plugin_xep_0054.py Fri Nov 12 17:21:24 2021 +0100 @@ -45,10 +45,7 @@ "Missing module pillow, please download/install it from https://python-pillow.github.io" ) -try: - from twisted.words.protocols.xmlstream import XMPPHandler -except ImportError: - from wokkel.subprotocols import XMPPHandler +from twisted.words.protocols.jabber.xmlstream import XMPPHandler IMPORT_NAME = "XEP-0054" @@ -109,7 +106,7 @@ NS_VCARD, client.profile) await client._xep_0054_avatar_hashes.load() - def savePhoto(self, client, photo_elt, entity_jid): + def savePhoto(self, client, photo_elt, entity): """Parse a <PHOTO> photo_elt and save the picture""" # XXX: this method is launched in a separate thread try: @@ -131,7 +128,7 @@ raise Failure(exceptions.NotFound()) if not buf: - log.warning("empty avatar for {jid}".format(jid=entity_jid.full())) + log.warning("empty avatar for {jid}".format(jid=entity.full())) raise Failure(exceptions.NotFound()) log.debug(_("Decoding binary")) @@ -140,7 +137,7 @@ if mime_type is None: log.debug( - f"no media type found specified for {entity_jid}'s avatar, trying to " + f"no media type found specified for {entity}'s avatar, trying to " f"guess") try: @@ -149,7 +146,7 @@ log.warning(f"Can't open avatar buffer: {e}") if mime_type is None: - msg = f"Can't find media type for {entity_jid}'s avatar" + msg = f"Can't find media type for {entity}'s avatar" log.warning(msg) raise Failure(exceptions.DataError(msg)) @@ -435,19 +432,6 @@ def getDiscoItems(self, requestor, target, nodeIdentifier=""): return [] - def _checkAvatarHash(self, client, entity, given_hash): - """Check that hash in cache (i.e. computed hash) is the same as given one""" - # XXX: if they differ, the avatar will be requested on each connection - # TODO: try to avoid re-requesting avatar in this case - computed_hash = client._xep_0054_avatar_hashes[entity.full()] - if computed_hash != given_hash: - log.warning( - "computed hash differs from given hash for {entity}:\n" - "computed: {computed}\ngiven: {given}".format( - entity=entity, computed=computed_hash, given=given_hash - ) - ) - async def update(self, presence): """Called on <presence/> stanza with vcard data @@ -468,19 +452,19 @@ except StopIteration: return - new_hash = str(photo_elt).strip() - if new_hash == HASH_SHA1_EMPTY: - new_hash = "" + given_hash = str(photo_elt).strip() + if given_hash == HASH_SHA1_EMPTY: + given_hash = "" hashes_cache = client._xep_0054_avatar_hashes old_hash = hashes_cache.get(entity_jid.full()) - if old_hash == new_hash: + if old_hash == given_hash: # no change, we can return… - if new_hash: + if given_hash: # …but we double check that avatar is in cache - avatar_cache = self.host.common_cache.getMetadata(new_hash) + avatar_cache = self.host.common_cache.getMetadata(given_hash) if avatar_cache is None: log.debug( f"Avatar for [{entity_jid}] is known but not in cache, we get " @@ -492,19 +476,19 @@ log.debug(f"avatar for {entity_jid} is already in cache") return - if new_hash is None: + if given_hash is None: # XXX: we use empty string to indicate that there is no avatar - new_hash = "" + given_hash = "" - await hashes_cache.aset(entity_jid.full(), new_hash) + await hashes_cache.aset(entity_jid.full(), given_hash) - if not new_hash: + if not given_hash: await self.plugin_parent._i.update( client, IMPORT_NAME, "avatar", None, entity_jid) # the avatar has been removed, no need to go further return - avatar_cache = self.host.common_cache.getMetadata(new_hash) + avatar_cache = self.host.common_cache.getMetadata(given_hash) if avatar_cache is not None: log.debug( f"New avatar found for [{entity_jid}], it's already in cache, we use it" @@ -516,7 +500,7 @@ 'path': avatar_cache['path'], 'filename': avatar_cache['filename'], 'media_type': avatar_cache['mime_type'], - 'cache_uid': new_hash, + 'cache_uid': given_hash, }, entity_jid ) @@ -528,7 +512,14 @@ if vcard is None: log.warning(f"Unexpected empty vCard for {entity_jid}") return - await self._checkAvatarHash(client, entity_jid, new_hash) + computed_hash = client._xep_0054_avatar_hashes[entity_jid.full()] + if computed_hash != given_hash: + log.warning( + "computed hash differs from given hash for {entity}:\n" + "computed: {computed}\ngiven: {given}".format( + entity=entity_jid, computed=computed_hash, given=given_hash + ) + ) def _update(self, presence): defer.ensureDeferred(self.update(presence))
--- a/sat/plugins/plugin_xep_0264.py Mon Sep 27 08:29:09 2021 +0200 +++ b/sat/plugins/plugin_xep_0264.py Fri Nov 12 17:21:24 2021 +0100 @@ -73,7 +73,10 @@ SIZE_MEDIUM = (640, 640) SIZE_BIG = (1280, 1280) SIZE_FULL_SCREEN = (2560, 2560) - SIZES = (SIZE_SMALL, SIZE_MEDIUM, SIZE_FULL_SCREEN) + # FIXME: SIZE_FULL_SCREEN is currently discarded as the resulting files are too big + # for BoB + # TODO: use an other mechanism than BoB for bigger files + SIZES = (SIZE_SMALL, SIZE_MEDIUM, SIZE_BIG) def __init__(self, host): log.info(_("Plugin XEP_0264 initialization"))
--- a/sat/plugins/plugin_xep_0384.py Mon Sep 27 08:29:09 2021 +0200 +++ b/sat/plugins/plugin_xep_0384.py Fri Nov 12 17:21:24 2021 +0100 @@ -1428,13 +1428,12 @@ header_elt = encrypted_elt.addElement('header') header_elt['sid'] = str(encryption_data['sid']) - for to_jid in to_jids: - bare_jid_s = to_jid.userhost() - - for rid, data in encryption_data['keys'][bare_jid_s].items(): + for key_data in encryption_data['keys'].values(): + for rid, data in key_data.items(): key_elt = header_elt.addElement( 'key', - content=b64enc(data['data'])) + content=b64enc(data['data']) + ) key_elt['rid'] = str(rid) if data['pre_key']: key_elt['prekey'] = 'true'
--- a/sat_frontends/jp/base.py Mon Sep 27 08:29:09 2021 +0200 +++ b/sat_frontends/jp/base.py Fri Nov 12 17:21:24 2021 +0100 @@ -121,33 +121,65 @@ return config.getConfig(self.sat_conf, section, name, default=default) def guess_background(self): - if not sys.stdin.isatty() or not sys.stdout.isatty(): - return 'dark' - stdin_fd = sys.stdin.fileno() - old_settings = termios.tcgetattr(stdin_fd) + # cf. https://unix.stackexchange.com/a/245568 (thanks!) try: - tty.setraw(sys.stdin.fileno()) - # we request background color - sys.stdout.write("\033]11;?\a") - sys.stdout.flush() - expected = "\033]11;rgb:" - for c in expected: - ch = sys.stdin.read(1) - if ch != c: - # background id is not supported, we default to "dark" - # TODO: log something? - return 'dark' - red, green, blue = [int(c, 16)/65535 for c in sys.stdin.read(14).split('/')] - # '\a' is the last character - sys.stdin.read(1) - finally: - termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_settings) + # for VTE based terminals + vte_version = int(os.getenv("VTE_VERSION", 0)) + except ValueError: + vte_version = 0 + + color_fg_bg = os.getenv("COLORFGBG") - lum = utils.per_luminance(red, green, blue) - if lum <= 0.5: - return 'dark' + if ((sys.stdin.isatty() and sys.stdout.isatty() + and ( + # XTerm + os.getenv("XTERM_VERSION") + # Konsole + or os.getenv("KONSOLE_VERSION") + # All VTE based terminals + or vte_version >= 3502 + ))): + # ANSI escape sequence + stdin_fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(stdin_fd) + try: + tty.setraw(sys.stdin.fileno()) + # we request background color + sys.stdout.write("\033]11;?\a") + sys.stdout.flush() + expected = "\033]11;rgb:" + for c in expected: + ch = sys.stdin.read(1) + if ch != c: + # background id is not supported, we default to "dark" + # TODO: log something? + return 'dark' + red, green, blue = [ + int(c, 16)/65535 for c in sys.stdin.read(14).split('/') + ] + # '\a' is the last character + sys.stdin.read(1) + finally: + termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_settings) + + lum = utils.per_luminance(red, green, blue) + if lum <= 0.5: + return 'dark' + else: + return 'light' + elif color_fg_bg: + # no luck with ANSI escape sequence, we try COLORFGBG environment variable + try: + bg = int(color_fg_bg.split(";")[-1]) + except ValueError: + return "dark" + if bg in list(range(7)) + [8]: + return "dark" + else: + return "light" else: - return 'light' + # no autodetection method found + return "dark" def set_color_theme(self): background = self.get_config('background', default='auto') @@ -159,6 +191,7 @@ 'your settings in libervia.conf').format( background=repr(background) )) + self.background = background if background == 'light': C.A_HEADER = A.FG_MAGENTA C.A_SUBHEADER = A.BOLD + A.FG_RED
--- a/sat_frontends/jp/cmd_debug.py Mon Sep 27 08:29:09 2021 +0200 +++ b/sat_frontends/jp/cmd_debug.py Fri Nov 12 17:21:24 2021 +0100 @@ -199,6 +199,7 @@ pass async def start(self): + print(f"background currently used: {A.BOLD}{self.host.background}{A.RESET}\n") for attr in dir(C): if not attr.startswith("A_"): continue
--- a/setup.py Mon Sep 27 08:29:09 2021 +0200 +++ b/setup.py Fri Nov 12 17:21:24 2021 +0100 @@ -41,7 +41,7 @@ 'pygments < 3', 'pygobject < 3.40.1', 'pyopenssl < 21.0.0', - 'python-dateutil < 3', + 'python-dateutil >= 2.8.1, < 3', 'python-potr < 1.1', 'pyxdg < 0.30', 'sat_tmp == 0.9.*',
--- a/tests/e2e/libervia-web/test_libervia-web.py Mon Sep 27 08:29:09 2021 +0200 +++ b/tests/e2e/libervia-web/test_libervia-web.py Fri Nov 12 17:21:24 2021 +0100 @@ -21,7 +21,7 @@ import pytest from helium import ( go_to, write, press, click, drag_file, find_all, wait_until, S, Text, Link, Button, - get_driver, ENTER + select, scroll_down, get_driver, ENTER ) @@ -178,3 +178,54 @@ click("test album") wait_until(lambda: not S("#loading_screen").exists()) assert len(find_all(S("img.is-photo-thumbnail"))) == 2 + + +class TestLists: + TEST_GENERIC_LIST_URL = ( + "https://libervia-web.test:8443/lists/view/account1@server1.test/" + "fdp%2Fsubmitted%2Forg.salut-a-toi.tickets%3A0_test-generic-list" + ) + + @pytest.mark.dependency(name="create_generic_list") + def test_user_can_create_generic(self, log_in_account1): + go_to("https://libervia-web.test:8443/lists/view") + click("create a list") + tickets_btn = S("//span[text()='Tickets']") + click(tickets_btn) + write("test-generic-list", "name of the list") + click("create list") + wait_until(Text("Success").exists) + + + @pytest.mark.dependency( + name="create_generic_list_item", depends=["create_generic_list"] + ) + def test_user_can_create_generic_list_item(self): + go_to(self.TEST_GENERIC_LIST_URL) + click("create") + write("test item", "title") + write("label 1, label 2", "labels") + select("type", "feature request") + write("this is a test body", "body") + scroll_down(600) + click("create list item") + wait_until(Text("Success").exists) + assert Link("create").exists() + assert Link("manage invitations").exists() + item = Link("test item") + assert item.exists() + labels_elt = item.web_element.find_element_by_class_name("xmlui_field__labels") + labels = [t.text for t in labels_elt.find_elements_by_tag_name("span")] + assert labels == ["label 1", "label 2"] + + @pytest.mark.dependency(depends=["create_generic_list_item"]) + def test_list_item_can_be_displayed(self): + go_to(self.TEST_GENERIC_LIST_URL) + item = Link("test item") + # FIXME: we can't click on the item created above with Selenium, looks like a + # Selenium bug, to be checked + go_to(item.href) + item_title = Text("test item") + assert item_title.exists() + item_body = Text("this is a test body") + assert item_body.exists()
--- a/tests/e2e/run_e2e.py Mon Sep 27 08:29:09 2021 +0200 +++ b/tests/e2e/run_e2e.py Fri Nov 12 17:21:24 2021 +0100 @@ -226,7 +226,7 @@ docker_compose.up("-d") p = docker_compose.exec( - "-T", "--workdir", "/src/libervia-backend/tests", "backend", + "--workdir", "/src/libervia-backend/tests", "backend", "pytest", "-o", "cache_dir=/tmp", *sys.argv[1:], color="yes", _in=sys.stdin, _out=live_out, _out_bufsize=0, _err=live_err, _err_bufsize=0, _bg=True