Mercurial > libervia-backend

comparison libervia/backend/plugins/plugin_comp_email_gateway/__init__.py @ 4337:95792a1f26c7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
component email gateway: attachments handling: attachments are now stored, and metadata are created in database. rel 453
author Goffi <goffi@goffi.org>
date Tue, 03 Dec 2024 00:13:23 +0100
parents 055930cc81f9
children 7c0b7ecb816f
comparison
equal deleted inserted replaced
4336:6e0918e638ee 4337:95792a1f26c7
19 from email.header import decode_header 19 from email.header import decode_header
20 from email.message import EmailMessage 20 from email.message import EmailMessage
21 from email.mime.text import MIMEText 21 from email.mime.text import MIMEText
22 from email.utils import formataddr, getaddresses, parseaddr 22 from email.utils import formataddr, getaddresses, parseaddr
23 from functools import partial 23 from functools import partial
24 import hashlib
25 from pathlib import Path
24 import re 26 import re
25 from typing import cast 27 import shutil
28 import tempfile
29 from typing import TYPE_CHECKING, NamedTuple, cast
26 30
27 from pydantic import BaseModel 31 from pydantic import BaseModel
28 from twisted.internet import defer, reactor 32 from twisted.internet import defer, reactor
33 from twisted.internet.threads import deferToThread
29 from twisted.mail import smtp 34 from twisted.mail import smtp
30 from twisted.words.protocols.jabber import jid 35 from twisted.words.protocols.jabber import jid
31 from twisted.words.protocols.jabber import error as jabber_error 36 from twisted.words.protocols.jabber import error as jabber_error
32 from twisted.words.protocols.jabber.error import StanzaError 37 from twisted.words.protocols.jabber.error import StanzaError
33 from twisted.words.protocols.jabber.xmlstream import XMPPHandler 38 from twisted.words.protocols.jabber.xmlstream import XMPPHandler
35 from wokkel import data_form, disco, iwokkel 40 from wokkel import data_form, disco, iwokkel
36 from zope.interface import implementer 41 from zope.interface import implementer
37 42
38 from libervia.backend.core import exceptions 43 from libervia.backend.core import exceptions
39 from libervia.backend.core.constants import Const as C 44 from libervia.backend.core.constants import Const as C
40 from libervia.backend.core.core_types import SatXMPPEntity 45 from libervia.backend.core.core_types import SatXMPPComponent, SatXMPPEntity
41 from libervia.backend.core.i18n import D_, _ 46 from libervia.backend.core.i18n import D_, _
42 from libervia.backend.core.log import getLogger 47 from libervia.backend.core.log import getLogger
43 from libervia.backend.memory.persistent import LazyPersistentBinaryDict 48 from libervia.backend.memory.persistent import LazyPersistentBinaryDict
44 from libervia.backend.memory.sqla import select 49 from libervia.backend.memory.sqla import select
45 from libervia.backend.memory.sqla_mapping import PrivateIndBin 50 from libervia.backend.memory.sqla_mapping import PrivateIndBin
46 from libervia.backend.models.core import MessageData 51 from libervia.backend.models.core import MessageData
47 from libervia.backend.plugins.plugin_xep_0033 import ( 52 from libervia.backend.plugins.plugin_xep_0033 import (
48 RECIPIENT_FIELDS,
49 AddressType, 53 AddressType,
50 AddressesData, 54 AddressesData,
55 RECIPIENT_FIELDS,
51 ) 56 )
52 from libervia.backend.plugins.plugin_xep_0077 import XEP_0077 57 from libervia.backend.plugins.plugin_xep_0077 import XEP_0077
53 from libervia.backend.plugins.plugin_xep_0106 import XEP_0106 58 from libervia.backend.plugins.plugin_xep_0106 import XEP_0106
54 from libervia.backend.plugins.plugin_xep_0131 import XEP_0131, HeadersData, Urgency 59 from libervia.backend.plugins.plugin_xep_0131 import HeadersData, Urgency, XEP_0131
60 from libervia.backend.plugins.plugin_xep_0498 import XEP_0498
55 from libervia.backend.tools.utils import aio 61 from libervia.backend.tools.utils import aio
56 62
63 from .imap import IMAPClientFactory
57 from .models import Credentials, UserData 64 from .models import Credentials, UserData
58 from .imap import IMAPClientFactory 65
66 if TYPE_CHECKING:
67 from libervia.backend.core.main import LiberviaBackend
59 68
60 69
61 log = getLogger(__name__) 70 log = getLogger(__name__)
62 71
63 IMPORT_NAME = "email-gateway" 72 IMPORT_NAME = "email-gateway"
67 C.PI_NAME: "Email Gateway Component", 76 C.PI_NAME: "Email Gateway Component",
68 C.PI_IMPORT_NAME: IMPORT_NAME, 77 C.PI_IMPORT_NAME: IMPORT_NAME,
69 C.PI_MODES: [C.PLUG_MODE_COMPONENT], 78 C.PI_MODES: [C.PLUG_MODE_COMPONENT],
70 C.PI_TYPE: C.PLUG_TYPE_ENTRY_POINT, 79 C.PI_TYPE: C.PLUG_TYPE_ENTRY_POINT,
71 C.PI_PROTOCOLS: [], 80 C.PI_PROTOCOLS: [],
72 C.PI_DEPENDENCIES: ["XEP-0033", "XEP-0077", "XEP-0106"], 81 C.PI_DEPENDENCIES: ["XEP-0033", "XEP-0077", "XEP-0106", "XEP-0498"],
73 C.PI_RECOMMENDATIONS: [], 82 C.PI_RECOMMENDATIONS: [],
74 C.PI_MAIN: "EmailGatewayComponent", 83 C.PI_MAIN: "EmailGatewayComponent",
75 C.PI_HANDLER: C.BOOL_TRUE, 84 C.PI_HANDLER: C.BOOL_TRUE,
76 C.PI_DESCRIPTION: D_( 85 C.PI_DESCRIPTION: D_(
77 "Gateway to handle email. Usual emails are handled as message, while mailing " 86 "Gateway to handle email. Usual emails are handled as message, while mailing "
84 KEY_CREDENTIALS = f"{PREFIX_KEY_CREDENTIALS}{{from_jid}}" 93 KEY_CREDENTIALS = f"{PREFIX_KEY_CREDENTIALS}{{from_jid}}"
85 94
86 email_pattern = re.compile(r"[^@]+@[^@]+\.[^@]+") 95 email_pattern = re.compile(r"[^@]+@[^@]+\.[^@]+")
87 96
88 97
98 class FileMetadata(NamedTuple):
99 path: Path
100 hash: str
101 size: int
102
103
89 class SendMailExtra(BaseModel): 104 class SendMailExtra(BaseModel):
90 addresses: AddressesData | None = None 105 addresses: AddressesData | None = None
91 headers: HeadersData | None = None 106 headers: HeadersData | None = None
92 107
93 108
94 class EmailGatewayComponent: 109 class EmailGatewayComponent:
95 IMPORT_NAME = IMPORT_NAME 110 IMPORT_NAME = IMPORT_NAME
96 verbose = 0 111 verbose = 0
97 112
98 def __init__(self, host): 113 def __init__(self, host: "LiberviaBackend") -> None:
99 self.host = host 114 self.host = host
100 self.client: SatXMPPEntity | None = None 115 self.client: SatXMPPComponent | None = None
101 self.initalized = False 116 self.initalized = False
102 self.storage: LazyPersistentBinaryDict | None = None 117 self.storage: LazyPersistentBinaryDict | None = None
103 self._iq_register = cast(XEP_0077, host.plugins["XEP-0077"]) 118 self._iq_register = cast(XEP_0077, host.plugins["XEP-0077"])
104 self._iq_register.register_handler( 119 self._iq_register.register_handler(
105 self._on_registration_form, self._on_registration_submit 120 self._on_registration_form, self._on_registration_submit
106 ) 121 )
107 self._e = cast(XEP_0106, host.plugins["XEP-0106"]) 122 self._e = cast(XEP_0106, host.plugins["XEP-0106"])
108 self._shim = cast(XEP_0131, host.plugins["XEP-0131"]) 123 self._shim = cast(XEP_0131, host.plugins["XEP-0131"])
124 self._pfs = cast(XEP_0498, host.plugins["XEP-0498"])
109 # TODO: For the moment, all credentials are kept in cache; we should only keep the 125 # TODO: For the moment, all credentials are kept in cache; we should only keep the
110 # X latest. 126 # X latest.
111 self.users_data: dict[jid.JID, UserData] = {} 127 self.users_data: dict[jid.JID, UserData] = {}
128 self.files_path = self.host.get_local_path(None, C.FILES_DIR)
112 host.trigger.add_with_check( 129 host.trigger.add_with_check(
113 "message_received", self, self._message_received_trigger, priority=-1000 130 "message_received", self, self._message_received_trigger, priority=-1000
114 ) 131 )
115 132
116 async def _init(self) -> None: 133 async def _init(self) -> None:
160 177
161 def get_handler(self, __) -> XMPPHandler: 178 def get_handler(self, __) -> XMPPHandler:
162 return EmailGatewayHandler() 179 return EmailGatewayHandler()
163 180
164 async def profile_connecting(self, client: SatXMPPEntity) -> None: 181 async def profile_connecting(self, client: SatXMPPEntity) -> None:
182 assert isinstance(client, SatXMPPComponent)
165 self.client = client 183 self.client = client
166 if not self.initalized: 184 if not self.initalized:
167 await self._init() 185 await self._init()
168 self.initalized = True 186 self.initalized = True
169 187
754 if headers: 772 if headers:
755 extra["headers"] = HeadersData(**headers).model_dump( 773 extra["headers"] = HeadersData(**headers).model_dump(
756 mode="json", exclude_none=True 774 mode="json", exclude_none=True
757 ) 775 )
758 776
777 # Handle attachments
778 for part in email.iter_attachments():
779 await self.handle_attachment(part, user_jid)
780
759 client = self.client.get_virtual_client(from_jid) 781 client = self.client.get_virtual_client(from_jid)
782
760 await client.sendMessage( 783 await client.sendMessage(
761 user_jid, 784 user_jid,
762 {"": body}, 785 {"": body},
763 {"": subject} if subject else None, 786 {"": subject} if subject else None,
764 extra=extra, 787 extra=extra,
765 ) 788 )
789
790 async def handle_attachment(self, part: EmailMessage, recipient_jid: jid.JID) -> None:
791 """Handle an attachment from an email.
792
793 @param part: The object representing the attachment.
794 @param recipient_jid: JID of the recipient to whom the attachment is being sent.
795 """
796 assert self.client is not None
797 content_type = part.get_content_type()
798 filename = part.get_filename() or "attachment"
799 log.debug(f"Handling attachment: {filename} ({content_type})")
800 file_metadata = await deferToThread(self._save_attachment, part)
801 if file_metadata is not None:
802 log.debug(f"Attachment {filename!r} saved to {file_metadata.path}")
803 try:
804 await self.host.memory.set_file(
805 self.client,
806 filename,
807 file_hash=file_metadata.hash,
808 hash_algo="sha-256",
809 size=file_metadata.size,
810 namespace=PLUGIN_INFO[C.PI_IMPORT_NAME],
811 mime_type=content_type,
812 owner=recipient_jid,
813 )
814 except Exception:
815 log.exception(f"Failed to register file {filename!r}")
816
817 def _save_attachment(self, part: EmailMessage) -> FileMetadata | None:
818 """Save the attachment to files path.
819
820 This method must be executed in a thread with deferToThread to avoid blocking the
821 reactor with IO operations if the attachment is large.
822
823 @param part: The object representing the attachment.
824 @return: Attachment data, or None if an error occurs.
825 @raises IOError: Can't save the attachment.
826 """
827 temp_file = None
828 try:
829 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
830 payload = part.get_payload(decode=True)
831 if isinstance(payload, bytes):
832 temp_file.write(payload)
833 file_hash = hashlib.sha256(payload).hexdigest()
834 file_path = self.files_path / file_hash
835 shutil.move(temp_file.name, file_path)
836 file_size = len(payload)
837 return FileMetadata(path=file_path, hash=file_hash, size=file_size)
838 else:
839 log.warning(f"Can't write payload of type {type(payload)}.")
840 return None
841 except Exception as e:
842 raise IOError(f"Failed to save attachment: {e}")
843 finally:
844 if temp_file is not None and Path(temp_file.name).exists():
845 Path(temp_file.name).unlink()
766 846
767 async def connect_imap(self, from_jid: jid.JID, user_data: UserData) -> None: 847 async def connect_imap(self, from_jid: jid.JID, user_data: UserData) -> None:
768 """Connect to IMAP service. 848 """Connect to IMAP service.
769 849
770 [self.on_new_email] will be used as callback on new messages. 850 [self.on_new_email] will be used as callback on new messages.