Mercurial > libervia-backend
annotate libervia/backend/plugins/plugin_comp_email_gateway/imap.py @ 4401:ae26233b655f default tip
doc (components): Add message cleaning section to email gateway doc:
fix 464
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Thu, 11 Sep 2025 21:17:51 +0200 |
| parents | 7ef21e3e5ac9 |
| children |
| rev | line source |
|---|---|
| 4303 | 1 #!/usr/bin/env python3 |
| 2 | |
| 3 # Libervia Email Gateway Component | |
| 4 # Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org) | |
| 5 | |
| 6 # This program is free software: you can redistribute it and/or modify | |
| 7 # it under the terms of the GNU Affero General Public License as published by | |
| 8 # the Free Software Foundation, either version 3 of the License, or | |
| 9 # (at your option) any later version. | |
| 10 | |
| 11 # This program is distributed in the hope that it will be useful, | |
| 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| 14 # GNU Affero General Public License for more details. | |
| 15 | |
| 16 # You should have received a copy of the GNU Affero General Public License | |
| 17 # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| 18 | |
|
4398
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
19 from collections.abc import Awaitable |
| 4303 | 20 from email.message import EmailMessage |
| 21 from email.parser import BytesParser, Parser | |
| 22 from email import policy | |
| 23 from typing import Callable, cast | |
| 24 from twisted.internet import defer, protocol, reactor | |
| 25 from twisted.internet.base import DelayedCall | |
| 26 from twisted.mail import imap4 | |
| 27 from twisted.python.failure import Failure | |
| 28 | |
| 29 from libervia.backend.core import exceptions | |
| 30 from libervia.backend.core.i18n import _ | |
| 31 from libervia.backend.core.log import getLogger | |
| 32 from .models import UserData | |
| 33 | |
| 34 log = getLogger(__name__) | |
| 35 | |
| 36 | |
| 37 class IMAPClient(imap4.IMAP4Client): | |
| 38 _idling = False | |
| 39 _idle_timer: DelayedCall | None = None | |
| 40 | |
| 41 def __init__(self, connected: defer.Deferred, *args, **kwargs) -> None: | |
| 42 super().__init__(*args, **kwargs) | |
| 43 self._connected = connected | |
| 44 | |
| 45 def serverGreeting(self, caps: dict) -> None: | |
| 46 """Handle the server greeting and capabilities. | |
| 47 | |
| 48 @param caps: Server capabilities. | |
| 49 """ | |
| 50 defer.ensureDeferred(self.on_server_greeting(caps)) | |
| 51 | |
| 52 async def on_server_greeting(self, caps: dict) -> None: | |
| 53 """Async method called when server greeting is received. | |
| 54 | |
| 55 @param caps: Server capabilities. | |
| 56 """ | |
| 57 self.server_capabilities = caps | |
| 58 try: | |
| 59 await self.authenticate(self.factory.password.encode()) | |
| 60 except Exception as e: | |
| 61 log.warning(f"Can't authenticate: {e}") | |
| 62 self._connected.errback( | |
| 63 exceptions.PasswordError("Authentication error for IMAP server.") | |
| 64 ) | |
| 65 return | |
| 66 log.debug("Authenticated.") | |
| 67 self._connected.callback(None) | |
| 68 if b"IDLE" in caps: | |
| 69 # We use "examine" for read-only access for now, will probably change in the | |
| 70 # future. | |
| 71 await self.examine(b"INBOX") | |
| 72 log.debug("Activating IDLE mode") | |
| 73 await self.idle() | |
| 74 else: | |
| 75 log.warning( | |
| 76 f'"IDLE" mode is not supported by your server, this gateways needs a ' | |
| 77 "server supporting this mode." | |
| 78 ) | |
| 79 return | |
| 80 | |
| 81 async def idle(self) -> None: | |
| 82 """Enter the IDLE mode to receive real-time updates from the server.""" | |
| 83 if self._idling: | |
| 84 # We are already in idle state. | |
| 85 return | |
| 86 self._idling = True | |
| 87 self._idle_timer = reactor.callLater(29 * 60, self.on_idle_timeout) | |
|
4398
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
88 log.debug("Starting IDLE mode.") |
| 4303 | 89 await self.sendCommand( |
| 90 imap4.Command( | |
| 91 b"IDLE", | |
| 92 continuation=lambda *a, **kw: log.debug(f"continuation: {a=} {kw=}"), | |
| 93 ) | |
| 94 ) | |
| 95 | |
| 96 def idle_exit(self) -> None: | |
| 97 """Exit the IDLE mode.""" | |
| 98 assert self._idling | |
| 99 assert self._idle_timer is not None | |
| 100 if not self._idle_timer.called: | |
| 101 self._idle_timer.cancel() | |
| 102 self._idle_timer = None | |
| 103 # Send DONE command to exit IDLE mode. | |
| 104 self.sendLine(b"DONE") | |
| 105 self._idling = False | |
|
4398
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
106 log.debug("IDLE mode terminated.") |
| 4303 | 107 |
| 108 def on_idle_timeout(self): | |
| 109 """Called when IDLE mode timeout is reached.""" | |
| 110 if self._idling: | |
| 111 # We've reached 29 min of IDLE mode, we restart it as recommended in the | |
| 112 # specifications. | |
| 113 self.idle_exit() | |
| 114 defer.ensureDeferred(self.idle()) | |
| 115 | |
| 116 def newMessages(self, exists: int | None, recent: int | None): | |
| 117 """Called when new messages are received. | |
| 118 | |
| 119 @param exists: Number of existing messages. | |
| 120 @param recent: Number of recent messages. | |
| 121 """ | |
| 122 defer.ensureDeferred(self.on_new_emails(exists, recent)) | |
| 123 | |
| 124 async def on_new_emails(self, exists: int | None, recent: int | None) -> None: | |
| 125 """Async method called when new messages are received. | |
| 126 | |
| 127 @param exists: Number of existing messages. | |
| 128 @param recent: Number of recent messages. | |
| 129 """ | |
| 130 log.debug(f"New messages: {exists}, Recent messages: {recent}") | |
|
4398
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
131 |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
132 if recent is None: |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
133 log.debug("No recent messages, skipping fetch.") |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
134 defer.ensureDeferred(self.idle()) |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
135 return |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
136 |
| 4303 | 137 self.idle_exit() |
|
4398
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
138 |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
139 # We retrieve recent messages. |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
140 recent_uids = await self.search(imap4.Query(recent=True), uid=True) |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
141 message_set = imap4.MessageSet() |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
142 for recent_uid in recent_uids: |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
143 message_set.add(recent_uid) |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
144 |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
145 try: |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
146 mess_data = await self.fetchMessage(message_set, uid=True) |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
147 |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
148 # Process all fetched messages. |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
149 log.debug(f"Fetched {len(mess_data)} messages.") |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
150 for message in mess_data.values(): |
| 4303 | 151 try: |
|
4398
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
152 content = message["RFC822"] |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
153 except KeyError: |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
154 log.warning(f"Can't find content for {message}.") |
| 4303 | 155 continue |
| 156 else: | |
|
4398
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
157 if isinstance(content, str): |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
158 parser = Parser(policy=policy.default) |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
159 parser_method = parser.parsestr |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
160 elif isinstance(content, bytes): |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
161 parser = BytesParser(policy=policy.default) |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
162 parser_method = parser.parsebytes |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
163 else: |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
164 log.error(f"Invalid content: {content}.") |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
165 continue |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
166 try: |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
167 parsed = parser_method(content) |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
168 except Exception as e: |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
169 log.warning(f"Can't parse content of email: {e}.") |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
170 continue |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
171 else: |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
172 assert self.factory is not None |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
173 factory = cast(IMAPClientFactory, self.factory) |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
174 await factory.on_new_email(parsed) |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
175 |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
176 except Exception as e: |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
177 log.error(f"Error fetching recent messages: {e}") |
| 4303 | 178 |
| 179 defer.ensureDeferred(self.idle()) | |
| 180 | |
| 181 def connectionLost(self, reason: Failure) -> None: | |
| 182 """Called when the connection is lost. | |
| 183 | |
| 184 @param reason: The reason for the lost connection. | |
| 185 """ | |
| 186 log.debug(f"connectionLost {reason=}") | |
| 187 if not self._connected.called: | |
| 188 self._connected.errback(reason) | |
| 189 super().connectionLost(reason) | |
| 190 | |
| 191 def lineReceived(self, line: bytes) -> None: | |
| 192 """Called when a line is received from the server. | |
| 193 | |
| 194 @param line: The received line. | |
| 195 """ | |
| 196 if self._idling: | |
| 197 if line == b"* OK Still here": | |
| 198 pass | |
| 199 elif line == b"+ idling": | |
| 200 pass | |
| 201 elif line.startswith(b"* "): | |
| 202 # Handle unsolicited responses during IDLE | |
| 203 self._extraInfo([imap4.parseNestedParens(line[2:])]) | |
| 204 else: | |
| 205 log.warning(f"Unexpected line received: {line!r}") | |
| 206 | |
| 207 return | |
| 208 | |
| 209 return super().lineReceived(line) | |
| 210 | |
| 211 def sendCommand(self, cmd: imap4.Command) -> defer.Deferred: | |
| 212 """Send a command to the server. | |
| 213 | |
| 214 This method is overriden to stop and restart IDLE mode when a command is received. | |
| 215 | |
| 216 @param cmd: The command to send. | |
| 217 @return: A deferred that fires when the command is sent. | |
| 218 """ | |
| 219 if self._idling and cmd.command != b"IDLE": | |
| 220 self.idle_exit() | |
| 221 d = super().sendCommand(cmd) | |
| 222 | |
| 223 def restart_idle_mode(ret): | |
| 224 defer.ensureDeferred(self.idle()) | |
| 225 return ret | |
| 226 | |
| 227 d.addCallback(restart_idle_mode) | |
| 228 return d | |
| 229 else: | |
| 230 return super().sendCommand(cmd) | |
| 231 | |
| 232 | |
| 233 class IMAPClientFactory(protocol.ClientFactory): | |
| 234 protocol = IMAPClient | |
| 235 | |
| 236 def __init__( | |
| 237 self, | |
| 238 user_data: UserData, | |
|
4398
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
239 on_new_email: Callable[[EmailMessage], Awaitable[None]], |
| 4303 | 240 connected: defer.Deferred, |
| 241 ) -> None: | |
| 242 """Initialize the IMAP client factory. | |
| 243 | |
|
4398
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
244 @param user_data: User data containing credentials and other user-specific |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
245 information. |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
246 @param on_new_email: Called when a new email is received. |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
247 @param connected: Deferred that will be fired when the IMAP connection is |
|
7ef21e3e5ac9
component email (imap): fetch all recent emails instead of only the last one.
Goffi <goffi@goffi.org>
parents:
4386
diff
changeset
|
248 established. |
| 4303 | 249 """ |
| 250 credentials = user_data.credentials | |
| 251 self.user_data = user_data | |
|
4386
c055042c01e3
component Email gateway: Convert mailing list to pubsub nodes:
Goffi <goffi@goffi.org>
parents:
4303
diff
changeset
|
252 self.username = credentials.imap_username |
|
c055042c01e3
component Email gateway: Convert mailing list to pubsub nodes:
Goffi <goffi@goffi.org>
parents:
4303
diff
changeset
|
253 self.password = credentials.imap_password |
| 4303 | 254 self.on_new_email = on_new_email |
| 255 self._connected = connected | |
| 256 | |
| 257 def buildProtocol(self, addr) -> IMAPClient: | |
| 258 """Build the IMAP client protocol. | |
| 259 | |
| 260 @return: The IMAP client protocol. | |
| 261 """ | |
| 262 assert self.protocol is not None | |
| 263 assert isinstance(self.protocol, type(IMAPClient)) | |
| 264 protocol_ = self.protocol(self._connected) | |
| 265 protocol_.factory = self | |
| 266 self.user_data.imap_client = protocol_ | |
| 267 assert isinstance(protocol_, IMAPClient) | |
| 268 protocol_.factory = self | |
| 269 encoded_username = self.username.encode() | |
| 270 | |
| 271 protocol_.registerAuthenticator(imap4.PLAINAuthenticator(encoded_username)) | |
| 272 protocol_.registerAuthenticator(imap4.LOGINAuthenticator(encoded_username)) | |
| 273 protocol_.registerAuthenticator( | |
| 274 imap4.CramMD5ClientAuthenticator(encoded_username) | |
| 275 ) | |
| 276 | |
| 277 return protocol_ | |
| 278 | |
| 279 def clientConnectionFailed(self, connector, reason: Failure) -> None: | |
| 280 """Called when the client connection fails. | |
| 281 | |
| 282 @param reason: The reason for the failure. | |
| 283 """ | |
| 284 log.warning(f"Connection failed: {reason}") |