--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/libervia/backend/plugins/plugin_comp_email_gateway/__init__.py	Fri Sep 06 18:07:17 2024 +0200
@@ -0,0 +1,621 @@
+#!/usr/bin/env python3
+
+# Libervia Email Gateway Component
+# Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from dataclasses import dataclass
+from email.header import decode_header
+from email.message import EmailMessage
+from email.mime.text import MIMEText
+from email.utils import formataddr, parseaddr
+from functools import partial
+import re
+from typing import Any, cast
+
+from twisted.internet import defer, reactor
+from twisted.mail import smtp
+from twisted.words.protocols.jabber import jid
+from twisted.words.protocols.jabber.error import StanzaError
+from twisted.words.protocols.jabber.xmlstream import XMPPHandler
+from twisted.words.xish import domish
+from wokkel import data_form, disco, iwokkel
+from zope.interface import implementer
+
+from libervia.backend.core import exceptions
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.core_types import SatXMPPEntity
+from libervia.backend.core.i18n import D_, _
+from libervia.backend.core.log import getLogger
+from libervia.backend.memory.persistent import LazyPersistentBinaryDict
+from libervia.backend.memory.sqla import select
+from libervia.backend.memory.sqla_mapping import PrivateIndBin
+from libervia.backend.models.core import MessageData
+from libervia.backend.plugins.plugin_xep_0077 import XEP_0077
+from libervia.backend.plugins.plugin_xep_0106 import XEP_0106
+from libervia.backend.tools.utils import aio
+
+from .models import Credentials, UserData
+from .imap import IMAPClientFactory
+
+
+log = getLogger(__name__)
+
+IMPORT_NAME = "email-gateway"
+NAME = "Libervia Email Gateway"
+
+PLUGIN_INFO = {
+    C.PI_NAME: "Email Gateway Component",
+    C.PI_IMPORT_NAME: IMPORT_NAME,
+    C.PI_MODES: [C.PLUG_MODE_COMPONENT],
+    C.PI_TYPE: C.PLUG_TYPE_ENTRY_POINT,
+    C.PI_PROTOCOLS: [],
+    C.PI_DEPENDENCIES: ["XEP-0077", "XEP-0106"],
+    C.PI_RECOMMENDATIONS: [],
+    C.PI_MAIN: "EmailGatewayComponent",
+    C.PI_HANDLER: C.BOOL_TRUE,
+    C.PI_DESCRIPTION: D_(
+        "Gateway to handle email. Usual emails are handled as message, while mailing "
+        "lists are converted to pubsub blogs."
+    ),
+}
+
+CONF_SECTION = f"component {IMPORT_NAME}"
+PREFIX_KEY_CREDENTIALS = "CREDENTIALS_"
+KEY_CREDENTIALS = f"{PREFIX_KEY_CREDENTIALS}{{from_jid}}"
+
+email_pattern = re.compile(r"[^@]+@[^@]+\.[^@]+")
+
+
+class EmailGatewayComponent:
+    IMPORT_NAME = IMPORT_NAME
+    verbose = 0
+
+    def __init__(self, host):
+        self.host = host
+        self.client: SatXMPPEntity | None = None
+        self.initalized = False
+        self.storage: LazyPersistentBinaryDict | None = None
+        self._iq_register = cast(XEP_0077, host.plugins["XEP-0077"])
+        self._iq_register.register_handler(
+            self._on_registration_form, self._on_registration_submit
+        )
+        self._e = cast(XEP_0106, host.plugins["XEP-0106"])
+        # TODO: For the moment, all credentials are kept in cache; we should only keep the
+        #   X latest.
+        self.users_data: dict[jid.JID, UserData] = {}
+        host.trigger.add_with_check(
+            "message_received", self, self._message_received_trigger, priority=-1000
+        )
+
+    async def _init(self) -> None:
+        """Initialisation done after profile is connected"""
+        assert self.client is not None
+        self.client.identities.append(disco.DiscoIdentity("gateway", "smtp", NAME))
+        self.storage = LazyPersistentBinaryDict(IMPORT_NAME, self.client.profile)
+        await self.connect_registered_users()
+
+    @aio
+    async def get_registered_users(self) -> dict[jid.JID, Credentials]:
+        """Retrieve credentials for all registered users
+
+        @return: a mapping from user JID to credentials data.
+        """
+        assert self.client is not None
+        profile_id = self.host.memory.storage.profiles[self.client.profile]
+        async with self.host.memory.storage.session() as session:
+            query = select(PrivateIndBin).where(
+                PrivateIndBin.profile_id == profile_id,
+                PrivateIndBin.namespace == IMPORT_NAME,
+                PrivateIndBin.key.startswith(PREFIX_KEY_CREDENTIALS),
+            )
+            result = await session.execute(query)
+            return {
+                jid.JID(p.key[len(PREFIX_KEY_CREDENTIALS) :]): p.value
+                for p in result.scalars()
+            }
+
+    async def connect_registered_users(self) -> None:
+        """Connected users already registered to the gateway."""
+        registered_data = await self.get_registered_users()
+        for user_jid, credentials in registered_data.items():
+            user_data = self.users_data[user_jid] = UserData(credentials=credentials)
+            if not credentials["imap_success"]:
+                log.warning(
+                    f"Ignoring unsuccessful IMAP credentials of {user_jid}. This user "
+                    "won't receive message from this gateway."
+                )
+            else:
+                try:
+                    await self.connect_imap(user_jid, user_data)
+                except Exception as e:
+                    log.warning(f"Can't connect {user_jid} to IMAP: {e}.")
+                else:
+                    log.debug(f"Connection to IMAP server successful for {user_jid}.")
+
+    def get_handler(self, __) -> XMPPHandler:
+        return EmailGatewayHandler()
+
+    async def profile_connecting(self, client: SatXMPPEntity) -> None:
+        self.client = client
+        if not self.initalized:
+            await self._init()
+            self.initalized = True
+
+    def _message_received_trigger(
+        self,
+        client: SatXMPPEntity,
+        message_elt: domish.Element,
+        post_treat: defer.Deferred,
+    ) -> bool:
+        """add the gateway workflow on post treatment"""
+        if client != self.client:
+            return True
+        post_treat.addCallback(
+            lambda mess_data: defer.ensureDeferred(
+                self.on_message(client, mess_data, message_elt)
+            )
+        )
+        return True
+
+    async def on_message(
+        self, client: SatXMPPEntity, mess_data: MessageData, message_elt: domish.Element
+    ) -> dict:
+        """Called once message has been parsed
+
+        @param client: Client session.
+        @param mess_data: Message data.
+        @return: Message data.
+        """
+        if client != self.client:
+            return mess_data
+        from_jid = mess_data["from"].userhostJID()
+        if mess_data["type"] not in ("chat", "normal"):
+            log.warning(f"ignoring message with unexpected type: {mess_data}")
+            return mess_data
+        if not client.is_local(from_jid):
+            log.warning(f"ignoring non local message: {mess_data}")
+            return mess_data
+        if not mess_data["to"].user:
+            log.warning(f"ignoring message addressed to gateway itself: {mess_data}")
+            return mess_data
+
+        try:
+            to_email = self._e.unescape(mess_data["to"].user)
+        except ValueError:
+            raise exceptions.DataError(
+                f'Invalid "to" JID, can\'t send message: {message_elt.toXml()}.'
+            )
+
+        try:
+            body_lang, body = next(iter(mess_data["message"].items()))
+        except (KeyError, StopIteration):
+            log.warning(f"No body found: {mess_data}")
+            body_lang, body = "", ""
+        try:
+            subject_lang, subject = next(iter(mess_data["subject"].items()))
+        except (KeyError, StopIteration):
+            subject_lang, subject = "", None
+
+        if not body and not subject:
+            log.warning(f"Ignoring empty message: {mess_data}")
+            return mess_data
+
+        try:
+            await self.send_email(
+                from_jid=from_jid,
+                to_email=to_email,
+                body=body,
+                subject=subject,
+            )
+        except exceptions.UnknownEntityError:
+            log.warning(f"Can't send message, user {from_jid} is not registered.")
+            message_error_elt = StanzaError(
+                "subscription-required",
+                text="User need to register to the gateway before sending emails.",
+            ).toResponse(message_elt)
+            await client.a_send(message_error_elt)
+
+            raise exceptions.CancelError("User not registered.")
+
+        return mess_data
+
+    async def send_email(
+        self,
+        from_jid: jid.JID,
+        to_email: str,
+        body: str,
+        subject: str | None,
+    ) -> None:
+        """Send an email using sender credentials.
+
+        Credentials will be retrieve from cache, or database.
+
+        @param from_jid: Bare JID of the sender.
+        @param to_email: Email address of the destinee.
+        @param body: Body of the email.
+        @param subject: Subject of the email.
+
+        @raise exceptions.UnknownEntityError: Credentials for "from_jid" can't be found.
+        """
+        # We need a bare jid.
+        assert self.storage is not None
+        assert not from_jid.resource
+        try:
+            user_data = self.users_data[from_jid]
+        except KeyError:
+            key = KEY_CREDENTIALS.format(from_jid=from_jid)
+            credentials = await self.storage.get(key)
+            if credentials is None:
+                raise exceptions.UnknownEntityError(
+                    f"No credentials found for {from_jid}."
+                )
+            self.users_data[from_jid] = UserData(credentials)
+        else:
+            credentials = user_data.credentials
+
+        msg = MIMEText(body, "plain", "UTF-8")
+        if subject is not None:
+            msg["Subject"] = subject
+        msg["From"] = formataddr(
+            (credentials["user_name"] or None, credentials["user_email"])
+        )
+        msg["To"] = to_email
+
+        sender_domain = credentials["user_email"].split("@", 1)[-1]
+
+        await smtp.sendmail(
+            credentials["smtp_host"].encode(),
+            credentials["user_email"].encode(),
+            [to_email.encode()],
+            msg.as_bytes(),
+            senderDomainName=sender_domain,
+            port=int(credentials["smtp_port"]),
+            username=credentials["smtp_username"].encode(),
+            password=credentials["smtp_password"].encode(),
+            requireAuthentication=True,
+            # TODO: only STARTTLS is supported right now, implicit TLS should be supported
+            #   too.
+            requireTransportSecurity=True,
+        )
+
+    async def _on_registration_form(
+        self, client: SatXMPPEntity, iq_elt: domish.Element
+    ) -> tuple[bool, data_form.Form] | None:
+        if client != self.client:
+            return
+        assert self.storage is not None
+        from_jid = jid.JID(iq_elt["from"])
+        key = KEY_CREDENTIALS.format(from_jid=from_jid.userhost())
+        credentials = await self.storage.get(key) or {}
+
+        form = data_form.Form(formType="form", title="IMAP/SMTP Credentials")
+
+        # Add instructions
+        form.instructions = [
+            D_(
+                "Please provide your IMAP and SMTP credentials to configure the "
+                "connection."
+            )
+        ]
+
+        # Add identity fields
+        form.addField(
+            data_form.Field(
+                fieldType="text-single",
+                var="user_name",
+                label="User Name",
+                desc=D_('The display name to use in the "From" field of sent emails.'),
+                value=credentials.get("user_name"),
+                required=True,
+            )
+        )
+
+        form.addField(
+            data_form.Field(
+                fieldType="text-single",
+                var="user_email",
+                label="User Email",
+                desc=D_('The email address to use in the "From" field of sent emails.'),
+                value=credentials.get("user_email"),
+                required=True,
+            )
+        )
+
+        # Add fields for IMAP credentials
+        form.addField(
+            data_form.Field(
+                fieldType="text-single",
+                var="imap_host",
+                label="IMAP Host",
+                desc=D_("IMAP server hostname or IP address"),
+                value=credentials.get("imap_host"),
+                required=True,
+            )
+        )
+        form.addField(
+            data_form.Field(
+                fieldType="text-single",
+                var="imap_port",
+                label="IMAP Port",
+                desc=D_("IMAP server port (default: 993)"),
+                value=credentials.get("imap_port", "993"),
+            )
+        )
+        form.addField(
+            data_form.Field(
+                fieldType="text-single",
+                var="imap_username",
+                label="IMAP Username",
+                desc=D_("Username for IMAP authentication"),
+                value=credentials.get("imap_username"),
+                required=True,
+            )
+        )
+        form.addField(
+            data_form.Field(
+                fieldType="text-private",
+                var="imap_password",
+                label="IMAP Password",
+                desc=D_("Password for IMAP authentication"),
+                value=credentials.get("imap_password"),
+                required=True,
+            )
+        )
+
+        # Add fields for SMTP credentials
+        form.addField(
+            data_form.Field(
+                fieldType="text-single",
+                var="smtp_host",
+                label="SMTP Host",
+                desc=D_("SMTP server hostname or IP address"),
+                value=credentials.get("smtp_host"),
+                required=True,
+            )
+        )
+        form.addField(
+            data_form.Field(
+                fieldType="text-single",
+                var="smtp_port",
+                label="SMTP Port",
+                desc=D_("SMTP server port (default: 587)"),
+                value=credentials.get("smtp_port", "587"),
+            )
+        )
+        form.addField(
+            data_form.Field(
+                fieldType="text-single",
+                var="smtp_username",
+                label="SMTP Username",
+                desc=D_("Username for SMTP authentication"),
+                value=credentials.get("smtp_username"),
+                required=True,
+            )
+        )
+        form.addField(
+            data_form.Field(
+                fieldType="text-private",
+                var="smtp_password",
+                label="SMTP Password",
+                desc=D_("Password for SMTP authentication"),
+                value=credentials.get("smtp_password"),
+                required=True,
+            )
+        )
+
+        return bool(credentials), form
+
+    def validate_field(
+        self,
+        form: data_form.Form,
+        key: str,
+        field_type: str,
+        min_value: int | None = None,
+        max_value: int | None = None,
+        default: str | int | None = None,
+    ) -> None:
+        """Validate a single field.
+
+        @param form: The form containing the fields.
+        @param key: The key of the field to validate.
+        @param field_type: The expected type of the field value.
+        @param min_value: Optional minimum value for integer fields.
+        @param max_value: Optional maximum value for integer fields.
+        @param default: Default value to use if the field is missing.
+        @raise StanzaError: If the field value is invalid or missing.
+        """
+        field = form.fields.get(key)
+        if field is None:
+            if default is None:
+                raise StanzaError("bad-request", text=f"{key} is required")
+            field = data_form.Field(var=key, value=str(default))
+            form.addField(field)
+
+        value = field.value
+        if field_type == "int":
+            try:
+                value = int(value)
+                if (min_value is not None and value < min_value) or (
+                    max_value is not None and value > max_value
+                ):
+                    raise ValueError
+            except (ValueError, TypeError):
+                raise StanzaError("bad-request", text=f"Invalid value for {key}: {value}")
+        elif field_type == "str":
+            if not isinstance(value, str):
+                raise StanzaError("bad-request", text=f"Invalid value for {key}: {value}")
+
+            # Basic email validation for user_email field
+            if key == "user_email":
+                # XXX: This is a minimal check. A complete email validation is notoriously
+                #   difficult.
+                if not email_pattern.match(value):
+                    raise StanzaError(
+                        "bad-request", text=f"Invalid email address: {value}"
+                    )
+
+    def validate_imap_smtp_form(self, submit_form: data_form.Form) -> None:
+        """Validate the submitted IMAP/SMTP credentials form.
+
+        @param submit_form: The submitted form containing IMAP/SMTP credentials.
+        @raise StanzaError: If any of the values are invalid.
+        """
+        # Validate identity fields
+        self.validate_field(submit_form, "user_name", "str")
+        self.validate_field(submit_form, "user_email", "str")
+
+        # Validate IMAP fields
+        self.validate_field(submit_form, "imap_host", "str")
+        self.validate_field(
+            submit_form, "imap_port", "int", min_value=1, max_value=65535, default=993
+        )
+        self.validate_field(submit_form, "imap_username", "str")
+        self.validate_field(submit_form, "imap_password", "str")
+
+        # Validate SMTP fields
+        self.validate_field(submit_form, "smtp_host", "str")
+        self.validate_field(
+            submit_form, "smtp_port", "int", min_value=1, max_value=65535, default=587
+        )
+        self.validate_field(submit_form, "smtp_username", "str")
+        self.validate_field(submit_form, "smtp_password", "str")
+
+    async def on_new_email(self, to_jid: jid.JID, email: EmailMessage) -> None:
+        """Called when a new message has been received.
+
+        @param to_jid: JID of the recipient.
+        @param email: Parsed email.
+        """
+        assert self.client is not None
+        name, email_addr = parseaddr(email["from"])
+        email_addr = email_addr.lower()
+        from_jid = jid.JID(None, (self._e.escape(email_addr), self.client.jid.host, None))
+
+        # Get the email body
+        body_mime = email.get_body(("plain",))
+        if body_mime is not None:
+            charset = body_mime.get_content_charset() or "utf-8"
+            body = body_mime.get_payload(decode=True).decode(charset, errors="replace")
+        else:
+            log.warning(f"No body found in email:\n{email}")
+            body = ""
+
+        # Decode the subject
+        subject = email.get("subject")
+        if subject:
+            decoded_subject = decode_header(subject)
+            subject = "".join(
+                [
+                    part.decode(encoding or "utf-8") if isinstance(part, bytes) else part
+                    for part, encoding in decoded_subject
+                ]
+            ).strip()
+        else:
+            subject = None
+
+        client = self.client.get_virtual_client(from_jid)
+        await client.sendMessage(to_jid, {"": body}, {"": subject} if subject else None)
+
+    async def connect_imap(self, from_jid: jid.JID, user_data: UserData) -> None:
+        """Connect to IMAP service.
+
+        [self.on_new_email] will be used as callback on new messages.
+
+        @param from_jid: JID of the user associated with given credentials.
+        @param credentials: Email credentials.
+        """
+        credentials = user_data.credentials
+
+        connected = defer.Deferred()
+        factory = IMAPClientFactory(
+            user_data,
+            partial(self.on_new_email, from_jid.userhostJID()),
+            connected,
+        )
+        reactor.connectTCP(
+            credentials["imap_host"], int(credentials["imap_port"]), factory
+        )
+        await connected
+
+    async def _on_registration_submit(
+        self,
+        client: SatXMPPEntity,
+        iq_elt: domish.Element,
+        submit_form: data_form.Form | None,
+    ) -> bool | None:
+        """Handle registration submit request.
+
+        Submit form is validated, and credentials are stored.
+        @param client: client session.
+        iq_elt: IQ stanza of the submission request.
+        submit_form: submit form.
+        @return: True if successful.
+            None if the callback is not relevant for this request.
+        """
+        if client != self.client:
+            return
+        assert self.storage is not None
+        from_jid = jid.JID(iq_elt["from"]).userhostJID()
+
+        if submit_form is None:
+            # This is an unregistration request.
+            try:
+                user_data = self.users_data[from_jid]
+            except KeyError:
+                pass
+            else:
+                if user_data.imap_client is not None:
+                    try:
+                        await user_data.imap_client.logout()
+                    except Exception:
+                        log.exception(f"Can't log out {from_jid} from IMAP server.")
+            key = KEY_CREDENTIALS.format(from_jid=from_jid)
+            await self.storage.adel(key)
+            log.info(f"{from_jid} unregistered from this gateway.")
+            return True
+
+        self.validate_imap_smtp_form(submit_form)
+        credentials = {key: field.value for key, field in submit_form.fields.items()}
+        user_data = self.users_data.get(from_jid)
+        if user_data is None:
+            # The user is not in cache, we cache current credentials.
+            user_data = self.users_data[from_jid] = UserData(credentials=credentials)
+        else:
+            # The user is known, we update credentials.
+            user_data.credentials = credentials
+        key = KEY_CREDENTIALS.format(from_jid=from_jid)
+        try:
+            await self.connect_imap(from_jid, user_data)
+        except Exception as e:
+            log.warning(f"Can't connect to IMAP server for {from_jid}")
+            credentials["imap_success"] = False
+            await self.storage.aset(key, credentials)
+            raise e
+        else:
+            log.debug(f"Connection successful to IMAP server for {from_jid}")
+            credentials["imap_success"] = True
+            await self.storage.aset(key, credentials)
+            return True
+
+
+@implementer(iwokkel.IDisco)
+class EmailGatewayHandler(XMPPHandler):
+
+    def getDiscoInfo(self, requestor, target, nodeIdentifier=""):
+        return []
+
+    def getDiscoItems(self, requestor, target, nodeIdentifier=""):
+        return []

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/libervia/backend/plugins/plugin_comp_email_gateway/imap.py	Fri Sep 06 18:07:17 2024 +0200
@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+
+# Libervia Email Gateway Component
+# Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from email.message import EmailMessage
+from email.parser import BytesParser, Parser
+from email import policy
+from typing import Callable, cast
+from twisted.internet import defer, protocol, reactor
+from twisted.internet.base import DelayedCall
+from twisted.mail import imap4
+from twisted.python.failure import Failure
+
+from libervia.backend.core import exceptions
+from libervia.backend.core.i18n import _
+from libervia.backend.core.log import getLogger
+from .models import UserData
+
+log = getLogger(__name__)
+
+
+class IMAPClient(imap4.IMAP4Client):
+    _idling = False
+    _idle_timer: DelayedCall | None = None
+
+    def __init__(self, connected: defer.Deferred, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._connected = connected
+
+    def serverGreeting(self, caps: dict) -> None:
+        """Handle the server greeting and capabilities.
+
+        @param caps: Server capabilities.
+        """
+        defer.ensureDeferred(self.on_server_greeting(caps))
+
+    async def on_server_greeting(self, caps: dict) -> None:
+        """Async method called when server greeting is received.
+
+        @param caps: Server capabilities.
+        """
+        self.server_capabilities = caps
+        try:
+            await self.authenticate(self.factory.password.encode())
+        except Exception as e:
+            log.warning(f"Can't authenticate: {e}")
+            self._connected.errback(
+                exceptions.PasswordError("Authentication error for IMAP server.")
+            )
+            return
+        log.debug("Authenticated.")
+        self._connected.callback(None)
+        if b"IDLE" in caps:
+            # We use "examine" for read-only access for now, will probably change in the
+            # future.
+            await self.examine(b"INBOX")
+            log.debug("Activating IDLE mode")
+            await self.idle()
+        else:
+            log.warning(
+                f'"IDLE" mode is not supported by your server, this gateways needs a '
+                "server supporting this mode."
+            )
+            return
+
+    async def idle(self) -> None:
+        """Enter the IDLE mode to receive real-time updates from the server."""
+        if self._idling:
+            # We are already in idle state.
+            return
+        self._idling = True
+        self._idle_timer = reactor.callLater(29 * 60, self.on_idle_timeout)
+        await self.sendCommand(
+            imap4.Command(
+                b"IDLE",
+                continuation=lambda *a, **kw: log.debug(f"continuation: {a=} {kw=}"),
+            )
+        )
+
+    def idle_exit(self) -> None:
+        """Exit the IDLE mode."""
+        assert self._idling
+        assert self._idle_timer is not None
+        if not self._idle_timer.called:
+            self._idle_timer.cancel()
+        self._idle_timer = None
+        # Send DONE command to exit IDLE mode.
+        self.sendLine(b"DONE")
+        self._idling = False
+        log.debug("IDLE mode terminated")
+
+    def on_idle_timeout(self):
+        """Called when IDLE mode timeout is reached."""
+        if self._idling:
+            # We've reached 29 min of IDLE mode, we restart it as recommended in the
+            # specifications.
+            self.idle_exit()
+            defer.ensureDeferred(self.idle())
+
+    def newMessages(self, exists: int | None, recent: int | None):
+        """Called when new messages are received.
+
+        @param exists: Number of existing messages.
+        @param recent: Number of recent messages.
+        """
+        defer.ensureDeferred(self.on_new_emails(exists, recent))
+
+    async def on_new_emails(self, exists: int | None, recent: int | None) -> None:
+        """Async method called when new messages are received.
+
+        @param exists: Number of existing messages.
+        @param recent: Number of recent messages.
+        """
+        log.debug(f"New messages: {exists}, Recent messages: {recent}")
+        log.debug("Retrieving last message.")
+        self.idle_exit()
+        mess_data = await self.fetchMessage("*")
+        for message in mess_data.values():
+            try:
+                content = message["RFC822"]
+            except KeyError:
+                log.warning(f"Can't find content for {message}.")
+                continue
+            else:
+                if isinstance(content, str):
+                    parser = Parser(policy=policy.default)
+                    parser_method = parser.parsestr
+                elif isinstance(content, bytes):
+                    parser = BytesParser(policy=policy.default)
+                    parser_method = parser.parsebytes
+                else:
+                    log.error(f"Invalid content: {content}")
+                    continue
+                try:
+                    parsed = parser_method(content)
+                except Exception as e:
+                    log.warning(f"Can't parse content of email: {e}")
+                    continue
+                else:
+                    assert self.factory is not None
+                    factory = cast(IMAPClientFactory, self.factory)
+                    await factory.on_new_email(parsed)
+
+        defer.ensureDeferred(self.idle())
+
+    def connectionLost(self, reason: Failure) -> None:
+        """Called when the connection is lost.
+
+        @param reason: The reason for the lost connection.
+        """
+        log.debug(f"connectionLost {reason=}")
+        if not self._connected.called:
+            self._connected.errback(reason)
+        super().connectionLost(reason)
+
+    def lineReceived(self, line: bytes) -> None:
+        """Called when a line is received from the server.
+
+        @param line: The received line.
+        """
+        if self._idling:
+            if line == b"* OK Still here":
+                pass
+            elif line == b"+ idling":
+                pass
+            elif line.startswith(b"* "):
+                # Handle unsolicited responses during IDLE
+                self._extraInfo([imap4.parseNestedParens(line[2:])])
+            else:
+                log.warning(f"Unexpected line received: {line!r}")
+
+            return
+
+        return super().lineReceived(line)
+
+    def sendCommand(self, cmd: imap4.Command) -> defer.Deferred:
+        """Send a command to the server.
+
+        This method is overriden to stop and restart IDLE mode when a command is received.
+
+        @param cmd: The command to send.
+        @return: A deferred that fires when the command is sent.
+        """
+        if self._idling and cmd.command != b"IDLE":
+            self.idle_exit()
+            d = super().sendCommand(cmd)
+
+            def restart_idle_mode(ret):
+                defer.ensureDeferred(self.idle())
+                return ret
+
+            d.addCallback(restart_idle_mode)
+            return d
+        else:
+            return super().sendCommand(cmd)
+
+
+class IMAPClientFactory(protocol.ClientFactory):
+    protocol = IMAPClient
+
+    def __init__(
+        self,
+        user_data: UserData,
+        on_new_email: Callable[[EmailMessage], None],
+        connected: defer.Deferred,
+    ) -> None:
+        """Initialize the IMAP client factory.
+
+        @param username: The username to use for authentication.
+        @param password: The password to use for authentication.
+        """
+        credentials = user_data.credentials
+        self.user_data = user_data
+        self.username = credentials["imap_username"]
+        self.password = credentials["imap_password"]
+        self.on_new_email = on_new_email
+        self._connected = connected
+
+    def buildProtocol(self, addr) -> IMAPClient:
+        """Build the IMAP client protocol.
+
+        @return: The IMAP client protocol.
+        """
+        assert self.protocol is not None
+        assert isinstance(self.protocol, type(IMAPClient))
+        protocol_ = self.protocol(self._connected)
+        protocol_.factory = self
+        self.user_data.imap_client = protocol_
+        assert isinstance(protocol_, IMAPClient)
+        protocol_.factory = self
+        encoded_username = self.username.encode()
+
+        protocol_.registerAuthenticator(imap4.PLAINAuthenticator(encoded_username))
+        protocol_.registerAuthenticator(imap4.LOGINAuthenticator(encoded_username))
+        protocol_.registerAuthenticator(
+            imap4.CramMD5ClientAuthenticator(encoded_username)
+        )
+
+        return protocol_
+
+    def clientConnectionFailed(self, connector, reason: Failure) -> None:
+        """Called when the client connection fails.
+
+        @param reason: The reason for the failure.
+        """
+        log.warning(f"Connection failed: {reason}")

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/libervia/backend/plugins/plugin_comp_email_gateway/models.py	Fri Sep 06 18:07:17 2024 +0200
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+# Libervia Email Gateway Component
+# Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from dataclasses import dataclass
+from typing import Any
+from twisted.mail import imap4
+
+
+Credentials = dict[str, Any]
+
+
+@dataclass
+class UserData:
+    credentials: Credentials
+    imap_client: imap4.IMAP4Client | None = None