Mercurial > libervia-backend

comparison sat/plugins/plugin_misc_maildir.py @ 2562:26edcf3a30eb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
core, setup: huge cleaning: - moved directories from src and frontends/src to sat and sat_frontends, which is the recommanded naming convention - move twisted directory to root - removed all hacks from setup.py, and added missing dependencies, it is now clean - use https URL for website in setup.py - removed "Environment :: X11 Applications :: GTK", as wix is deprecated and removed - renamed sat.sh to sat and fixed its installation - added python_requires to specify Python version needed - replaced glib2reactor which use deprecated code by gtk3reactor sat can now be installed directly from virtualenv without using --system-site-packages anymore \o/
author Goffi <goffi@goffi.org>
date Mon, 02 Apr 2018 19:44:50 +0200
parents src/plugins/plugin_misc_maildir.py@33c8c4973743
children ab2696e34d29
comparison
equal deleted inserted replaced
2561:bd30dc3ffe5a 2562:26edcf3a30eb
1 #!/usr/bin/env python2
2 # -*- coding: utf-8 -*-
3
4 # SàT plugin for managing Maildir type mail boxes
5 # Copyright (C) 2011 Jérôme Poisson (goffi@goffi.org)
6
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
11
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
16
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19
20 from sat.core.i18n import D_, _
21 from sat.core.constants import Const as C
22 from sat.core.log import getLogger
23 log = getLogger(__name__)
24 import warnings
25 warnings.filterwarnings('ignore', 'the MimeWriter', DeprecationWarning, 'twisted') # FIXME: to be removed, see http://twistedmatrix.com/trac/ticket/4038
26 from twisted.mail import maildir
27 import email.message
28 import email.utils
29 import os
30 from sat.core.exceptions import ProfileUnknownError
31 from sat.memory.persistent import PersistentBinaryDict
32
33
34 PLUGIN_INFO = {
35 C.PI_NAME: "Maildir Plugin",
36 C.PI_IMPORT_NAME: "Maildir",
37 C.PI_TYPE: "Misc",
38 C.PI_PROTOCOLS: [],
39 C.PI_DEPENDENCIES: [],
40 C.PI_MAIN: "MaildirBox",
41 C.PI_HANDLER: "no",
42 C.PI_DESCRIPTION: _("""Intercept "normal" type messages, and put them in a Maildir type box""")
43 }
44
45 MAILDIR_PATH = "Maildir"
46 CATEGORY = D_("Mail Server")
47 NAME = D_('Block "normal" messages propagation')
48 # FIXME: (very) old and (very) experimental code, need a big cleaning/review or to be deprecated
49
50
51 class MaildirError(Exception):
52 pass
53
54
55 class MaildirBox(object):
56 params = """
57 <params>
58 <individual>
59 <category name='{category_name}' label='{category_label}'>
60 <param name='{name}' label='{label}' value="false" type="bool" security="4" />
61 </category>
62 </individual>
63 </params>
64 """.format(category_name=CATEGORY,
65 category_label=_(CATEGORY),
66 name=NAME,
67 label=_(NAME),
68 )
69
70 def __init__(self, host):
71 log.info(_("Plugin Maildir initialization"))
72 self.host = host
73 host.memory.updateParams(self.params)
74
75 self.__observed = {}
76 self.data = {} # list of profile spectific data. key = profile, value = PersistentBinaryDict where key=mailbox name,
77 # and value is a dictionnary with the following value
78 # - cur_idx: value of the current unique integer increment (UID)
79 # - message_id (as returned by MaildirMailbox): a tuple of (UID, [flag1, flag2, ...])
80 self.__mailboxes = {} # key: profile, value: {boxname: MailboxUser instance}
81
82 #the triggers
83 host.trigger.add("MessageReceived", self.messageReceivedTrigger)
84
85 def profileConnected(self, client):
86 """Called on client connection, create profile data"""
87 profile = client.profile
88 self.data[profile] = PersistentBinaryDict("plugin_maildir", profile)
89 self.__mailboxes[profile] = {}
90
91 def dataLoaded(ignore):
92 if not self.data[profile]:
93 #the mailbox is new, we initiate the data
94 self.data[profile]["INBOX"] = {"cur_idx": 0}
95 self.data[profile].load().addCallback(dataLoaded)
96
97 def profileDisconnected(self, client):
98 """Called on profile disconnection, free profile's resources"""
99 profile = client.profile
100 del self.__mailboxes[profile]
101 del self.data[profile]
102
103 def messageReceivedTrigger(self, client, message, post_treat):
104 """This trigger catch normal message and put the in the Maildir box.
105 If the message is not of "normal" type, do nothing
106 @param message: message xmlstrem
107 @return: False if it's a normal message, True else"""
108 profile = client.profile
109 for e in message.elements(C.NS_CLIENT, 'body'):
110 mess_type = message.getAttribute('type', 'normal')
111 if mess_type != 'normal':
112 return True
113 self.accessMessageBox("INBOX", profile_key=profile).addMessage(message)
114 return not self.host.memory.getParamA(NAME, CATEGORY, profile_key=profile)
115 return True
116
117 def accessMessageBox(self, boxname, observer=None, profile_key=C.PROF_KEY_NONE):
118 """Create and return a MailboxUser instance
119 @param boxname: name of the box
120 @param observer: method to call when a NewMessage arrive"""
121 profile = self.host.memory.getProfileName(profile_key)
122 if not profile:
123 raise ProfileUnknownError(profile_key)
124 if boxname not in self.__mailboxes[profile]:
125 self.__mailboxes[profile][boxname] = MailboxUser(self, boxname, observer, profile=profile)
126 else:
127 if observer:
128 self.addObserver(observer, profile, boxname)
129 return self.__mailboxes[profile][boxname]
130
131 def _getProfilePath(self, profile):
132 """Return a unique path for profile's mailbox
133 The path must be unique, usable as a dir name, and bijectional"""
134 return profile.replace('/', '_').replace('..', '_') # FIXME: this is too naive to work well, must be improved
135
136 def _removeBoxAccess(self, boxname, mailboxUser, profile):
137 """Remove a reference to a box
138 @param name: name of the box
139 @param mailboxUser: MailboxUser instance"""
140 if boxname not in self.__mailboxes:
141 err_msg = _("Trying to remove an mailboxUser not referenced")
142 log.error(_(u"INTERNAL ERROR: ") + err_msg)
143 raise MaildirError(err_msg)
144 assert self.__mailboxes[profile][boxname] == mailboxUser
145 del self.__mailboxes[profile][boxname]
146
147 def _checkBoxReference(self, boxname, profile):
148 """Check if there is a reference on a box, and return it
149 @param boxname: name of the box to check
150 @return: MailboxUser instance or None"""
151 if profile in self.__mailboxes:
152 if boxname in self.__mailboxes[profile]:
153 return self.__mailboxes[profile][boxname]
154
155 def __getBoxData(self, boxname, profile):
156 """Return the date of a box"""
157 try:
158 return self.data[profile][boxname] # the boxname MUST exist in the data
159 except KeyError:
160 err_msg = _("Boxname doesn't exist in internal data")
161 log.error(_(u"INTERNAL ERROR: ") + err_msg)
162 raise MaildirError(err_msg)
163
164 def getUid(self, boxname, message_id, profile):
165 """Return an unique integer, always ascending, for a message
166 This is mainly needed for the IMAP protocol
167 @param boxname: name of the box where the message is
168 @param message_id: unique id of the message as given by MaildirMailbox
169 @return: Integer UID"""
170 box_data = self.__getBoxData(boxname, profile)
171 if message_id in box_data:
172 ret = box_data[message_id][0]
173 else:
174 box_data['cur_idx'] += 1
175 box_data[message_id] = [box_data['cur_idx'], []]
176 ret = box_data[message_id]
177 self.data[profile].force(boxname)
178 return ret
179
180 def getNextUid(self, boxname, profile):
181 """Return next unique integer that will generated
182 This is mainly needed for the IMAP protocol
183 @param boxname: name of the box where the message is
184 @return: Integer UID"""
185 box_data = self.__getBoxData(boxname, profile)
186 return box_data['cur_idx'] + 1
187
188 def getNextExistingUid(self, boxname, uid, profile):
189 """Give the next uid of existing message
190 @param boxname: name of the box where the message is
191 @param uid: uid to start from
192 @return: uid or None if the is no more message"""
193 box_data = self.__getBoxData(boxname, profile)
194 idx = uid + 1
195 while self.getIdFromUid(boxname, idx, profile) is None: # TODO: this is highly inefficient because getIdfromUid is inefficient, fix this
196 idx += 1
197 if idx > box_data['cur_idx']:
198 return None
199 return idx
200
201 def getMaxUid(self, boxname, profile):
202 """Give the max existing uid
203 @param boxname: name of the box where the message is
204 @return: uid"""
205 box_data = self.__getBoxData(boxname, profile)
206 return box_data['cur_idx']
207
208 def getIdFromUid(self, boxname, message_uid, profile):
209 """Return the message unique id from it's integer UID
210 @param boxname: name of the box where the message is
211 @param message_uid: unique integer identifier
212 @return: unique id of the message as given by MaildirMailbox or None if not found"""
213 box_data = self.__getBoxData(boxname, profile)
214 for message_id in box_data.keys(): # TODO: this is highly inefficient on big mailbox, must be replaced in the future
215 if message_id == 'cur_idx':
216 continue
217 if box_data[message_id][0] == message_uid:
218 return message_id
219 return None
220
221 def getFlags(self, boxname, mess_id, profile):
222 """Return the messages flags
223 @param boxname: name of the box where the message is
224 @param message_idx: message id as given by MaildirMailbox
225 @return: list of strings"""
226 box_data = self.__getBoxData(boxname, profile)
227 if mess_id not in box_data:
228 raise MaildirError("Trying to get flags from an unexisting message")
229 return box_data[mess_id][1]
230
231 def setFlags(self, boxname, mess_id, flags, profile):
232 """Change the flags of the message
233 @param boxname: name of the box where the message is
234 @param message_idx: message id as given by MaildirMailbox
235 @param flags: list of strings
236 """
237 box_data = self.__getBoxData(boxname, profile)
238 assert(type(flags) == list)
239 flags = [flag.upper() for flag in flags] # we store every flag UPPERCASE
240 if mess_id not in box_data:
241 raise MaildirError("Trying to set flags for an unexisting message")
242 box_data[mess_id][1] = flags
243 self.data[profile].force(boxname)
244
245 def getMessageIdsWithFlag(self, boxname, flag, profile):
246 """Return ids of messages where a flag is set
247 @param boxname: name of the box where the message is
248 @param flag: flag to check
249 @return: list of id (as given by MaildirMailbox)"""
250 box_data = self.__getBoxData(boxname, profile)
251 assert(isinstance(flag, basestring))
252 flag = flag.upper()
253 result = []
254 for key in box_data:
255 if key == 'cur_idx':
256 continue
257 if flag in box_data[key][1]:
258 result.append(key)
259 return result
260
261 def purgeDeleted(self, boxname, profile):
262 """Remove data for messages with flag "\\Deleted"
263 @param boxname: name of the box where the message is
264 """
265 box_data = self.__getBoxData(boxname, profile)
266 for mess_id in self.getMessageIdsWithFlag(boxname, "\\Deleted", profile):
267 del(box_data[mess_id])
268 self.data[profile].force(boxname)
269
270 def cleanTable(self, boxname, existant_id, profile):
271 """Remove mails which no longuer exist from the table
272 @param boxname: name of the box to clean
273 @param existant_id: list of id which actually exist"""
274 box_data = self.__getBoxData(boxname, profile)
275 to_remove = []
276 for key in box_data:
277 if key not in existant_id and key != "cur_idx":
278 to_remove.append(key)
279 for key in to_remove:
280 del box_data[key]
281
282 def addObserver(self, callback, profile, boxname, signal="NEW_MESSAGE"):
283 """Add an observer for maildir box changes
284 @param callback: method to call when the the box is updated
285 @param boxname: name of the box to observe
286 @param signal: which signal is observed by the caller"""
287 if (profile, boxname) not in self.__observed:
288 self.__observed[(profile, boxname)] = {}
289 if signal not in self.__observed[(profile, boxname)]:
290 self.__observed[(profile, boxname)][signal] = set()
291 self.__observed[(profile, boxname)][signal].add(callback)
292
293 def removeObserver(self, callback, profile, boxname, signal="NEW_MESSAGE"):
294 """Remove an observer of maildir box changes
295 @param callback: method to remove from obervers
296 @param boxname: name of the box which was observed
297 @param signal: which signal was observed by the caller"""
298 if (profile, boxname) not in self.__observed:
299 err_msg = _(u"Trying to remove an observer for an inexistant mailbox")
300 log.error(_(u"INTERNAL ERROR: ") + err_msg)
301 raise MaildirError(err_msg)
302 if signal not in self.__observed[(profile, boxname)]:
303 err_msg = _(u"Trying to remove an inexistant observer, no observer for this signal")
304 log.error(_(u"INTERNAL ERROR: ") + err_msg)
305 raise MaildirError(err_msg)
306 if not callback in self.__observed[(profile, boxname)][signal]:
307 err_msg = _(u"Trying to remove an inexistant observer")
308 log.error(_(u"INTERNAL ERROR: ") + err_msg)
309 raise MaildirError(err_msg)
310 self.__observed[(profile, boxname)][signal].remove(callback)
311
312 def emitSignal(self, profile, boxname, signal_name):
313 """Emit the signal to observer"""
314 log.debug(u'emitSignal %s %s %s' % (profile, boxname, signal_name))
315 try:
316 for observer_cb in self.__observed[(profile, boxname)][signal_name]:
317 observer_cb()
318 except KeyError:
319 pass
320
321
322 class MailboxUser(object):
323 """This class is used to access a mailbox"""
324
325 def xmppMessage2mail(self, message):
326 """Convert the XMPP's XML message to a basic rfc2822 message
327 @param xml: domish.Element of the message
328 @return: string email"""
329 mail = email.message.Message()
330 mail['MIME-Version'] = "1.0"
331 mail['Content-Type'] = "text/plain; charset=UTF-8; format=flowed"
332 mail['Content-Transfer-Encoding'] = "8bit"
333 mail['From'] = message['from'].encode('utf-8')
334 mail['To'] = message['to'].encode('utf-8')
335 mail['Date'] = email.utils.formatdate().encode('utf-8')
336 #TODO: save thread id
337 for e in message.elements():
338 if e.name == "body":
339 mail.set_payload(e.children[0].encode('utf-8'))
340 elif e.name == "subject":
341 mail['Subject'] = e.children[0].encode('utf-8')
342 return mail.as_string()
343
344 def __init__(self, _maildir, name, observer=None, profile=C.PROF_KEY_NONE):
345 """@param _maildir: the main MaildirBox instance
346 @param name: name of the mailbox
347 @param profile: real profile (ie not a profile_key)
348 THIS OBJECT MUST NOT BE USED DIRECTLY: use MaildirBox.accessMessageBox instead"""
349 if _maildir._checkBoxReference(name, profile):
350 log.error(u"INTERNAL ERROR: MailboxUser MUST NOT be instancied directly")
351 raise MaildirError('double MailboxUser instanciation')
352 if name != "INBOX":
353 raise NotImplementedError
354 self.name = name
355 self.profile = profile
356 self.maildir = _maildir
357 profile_path = self.maildir._getProfilePath(profile)
358 full_profile_path = os.path.join(self.maildir.host.memory.getConfig('', 'local_dir'), 'maildir', profile_path)
359 if not os.path.exists(full_profile_path):
360 os.makedirs(full_profile_path, 0700)
361 mailbox_path = os.path.join(full_profile_path, MAILDIR_PATH)
362 self.mailbox_path = mailbox_path
363 self.mailbox = maildir.MaildirMailbox(mailbox_path)
364 self.observer = observer
365 self.__uid_table_update()
366
367 if observer:
368 log.debug(u"adding observer for %s (%s)" % (name, profile))
369 self.maildir.addObserver(observer, profile, name, "NEW_MESSAGE")
370
371 def __uid_table_update(self):
372 existant_id = []
373 for mess_idx in range(self.getMessageCount()):
374 #we update the uid table
375 existant_id.append(self.getId(mess_idx))
376 self.getUid(mess_idx)
377 self.maildir.cleanTable(self.name, existant_id, profile=self.profile)
378
379 def __del__(self):
380 if self.observer:
381 log.debug(u"removing observer for %s" % self.name)
382 self._maildir.removeObserver(self.observer, self.name, "NEW_MESSAGE")
383 self.maildir._removeBoxAccess(self.name, self, profile=self.profile)
384
385 def addMessage(self, message):
386 """Add a message to the box
387 @param message: XMPP XML message"""
388 self.mailbox.appendMessage(self.xmppMessage2mail(message)).addCallback(self.emitSignal, "NEW_MESSAGE")
389
390 def emitSignal(self, ignore, signal):
391 """Emit the signal to the observers"""
392 if signal == "NEW_MESSAGE":
393 self.getUid(self.getMessageCount() - 1) # XXX: we make an uid for the last message added
394 self.maildir.emitSignal(self.profile, self.name, signal)
395
396 def getId(self, mess_idx):
397 """Return the Unique ID of the message
398 @mess_idx: message index"""
399 return self.mailbox.getUidl(mess_idx)
400
401 def getUid(self, mess_idx):
402 """Return a unique interger id for the message, always ascending"""
403 mess_id = self.getId(mess_idx)
404 return self.maildir.getUid(self.name, mess_id, profile=self.profile)
405
406 def getNextUid(self):
407 return self.maildir.getNextUid(self.name, profile=self.profile)
408
409 def getNextExistingUid(self, uid):
410 return self.maildir.getNextExistingUid(self.name, uid, profile=self.profile)
411
412 def getMaxUid(self):
413 return self.maildir.getMaxUid(self.name, profile=self.profile)
414
415 def getMessageCount(self):
416 """Return number of mails present in this box"""
417 return len(self.mailbox.list)
418
419 def getMessageIdx(self, mess_idx):
420 """Return the full message
421 @mess_idx: message index"""
422 return self.mailbox.getMessage(mess_idx)
423
424 def getIdxFromUid(self, mess_uid):
425 """Return the message index from the uid
426 @param mess_uid: message unique identifier
427 @return: message index, as managed by MaildirMailbox"""
428 for mess_idx in range(self.getMessageCount()):
429 if self.getUid(mess_idx) == mess_uid:
430 return mess_idx
431 raise IndexError
432
433 def getIdxFromId(self, mess_id):
434 """Return the message index from the unique index
435 @param mess_id: message unique index as given by MaildirMailbox
436 @return: message sequence index"""
437 for mess_idx in range(self.getMessageCount()):
438 if self.mailbox.getUidl(mess_idx) == mess_id:
439 return mess_idx
440 raise IndexError
441
442 def getMessage(self, mess_idx):
443 """Return the full message
444 @param mess_idx: message index"""
445 return self.mailbox.getMessage(mess_idx)
446
447 def getMessageUid(self, mess_uid):
448 """Return the full message
449 @param mess_idx: message unique identifier"""
450 return self.mailbox.getMessage(self.getIdxFromUid(mess_uid))
451
452 def getFlags(self, mess_idx):
453 """Return the flags of the message
454 @param mess_idx: message index
455 @return: list of strings"""
456 id = self.getId(mess_idx)
457 return self.maildir.getFlags(self.name, id, profile=self.profile)
458
459 def getFlagsUid(self, mess_uid):
460 """Return the flags of the message
461 @param mess_uid: message unique identifier
462 @return: list of strings"""
463 id = self.maildir.getIdFromUid(self.name, mess_uid, profile=self.profile)
464 return self.maildir.getFlags(self.name, id, profile=self.profile)
465
466 def setFlags(self, mess_idx, flags):
467 """Change the flags of the message
468 @param mess_idx: message index
469 @param flags: list of strings
470 """
471 id = self.getId(mess_idx)
472 self.maildir.setFlags(self.name, id, flags, profile=self.profile)
473
474 def setFlagsUid(self, mess_uid, flags):
475 """Change the flags of the message
476 @param mess_uid: message unique identifier
477 @param flags: list of strings
478 """
479 id = self.maildir.getIdFromUid(self.name, mess_uid, profile=self.profile)
480 return self.maildir.setFlags(self.name, id, flags, profile=self.profile)
481
482 def getMessageIdsWithFlag(self, flag):
483 """Return ids of messages where a flag is set
484 @param flag: flag to check
485 @return: list of id (as given by MaildirMailbox)"""
486 return self.maildir.getMessageIdsWithFlag(self.name, flag, profile=self.profile)
487
488 def removeDeleted(self):
489 """Actually delete message flagged "\\Deleted"
490 Also purge the internal data of these messages
491 """
492 for mess_id in self.getMessageIdsWithFlag("\\Deleted"):
493 print ("Deleting %s" % mess_id)
494 self.mailbox.deleteMessage(self.getIdxFromId(mess_id))
495 self.mailbox = maildir.MaildirMailbox(self.mailbox_path) # We need to reparse the dir to have coherent indexing
496 self.maildir.purgeDeleted(self.name, profile=self.profile)
497
498 def emptyTrash(self):
499 """Delete everything in the .Trash dir"""
500 pass #TODO