From ecdb4ac880739077a97b8a7a60ed8aa635cd401c Mon Sep 17 00:00:00 2001 From: Matthieu Dietrich Date: Mon, 20 Jul 2015 16:11:28 +0200 Subject: [PATCH 1/7] Add new module "mail_cleanup" in order to move/mark as read old messages --- mail_cleanup/README.rst | 63 ++++++++++++++++ mail_cleanup/__init__.py | 1 + mail_cleanup/__openerp__.py | 88 +++++++++++++++++++++++ mail_cleanup/mail_cleanup.py | 135 +++++++++++++++++++++++++++++++++++ mail_cleanup/mail_view.xml | 21 ++++++ 5 files changed, 308 insertions(+) create mode 100644 mail_cleanup/README.rst create mode 100644 mail_cleanup/__init__.py create mode 100644 mail_cleanup/__openerp__.py create mode 100644 mail_cleanup/mail_cleanup.py create mode 100644 mail_cleanup/mail_view.xml diff --git a/mail_cleanup/README.rst b/mail_cleanup/README.rst new file mode 100644 index 000000000..161ae6eae --- /dev/null +++ b/mail_cleanup/README.rst @@ -0,0 +1,63 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License: AGPL-3 + +Mail cleanup +=========== + +This module allows to mark e-mails older than x days as read +and optionally move them on IMAP servers, just before fetching them. + +Since the main "mail" module does not mark unroutable e-mails as read, +this means that if junk mail arrives in the catch-all address without +any default route, fetching newer e-mails will happen after re-parsing +those unroutable e-mails. + +Configuration +============= + +This module depends on ``mail_environment`` in order to add "expiration dates" +per server. + +Example of a configuration file (add those values to your server):: + + [incoming_mail.openerp_pop_mail1] + cleanup_days = 30 # default value + cleanup_folder = NotParsed # optional parameter + +Known issues / Roadmap +====================== + +* None + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed feedback +`here `_. + + +Credits +======= + +Contributors +------------ + +* Matthieu Dietrich + +Maintainer +---------- + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit http://odoo-community.org. + diff --git a/mail_cleanup/__init__.py b/mail_cleanup/__init__.py new file mode 100644 index 000000000..cc2c6b2e5 --- /dev/null +++ b/mail_cleanup/__init__.py @@ -0,0 +1 @@ +from . import mail_cleanup diff --git a/mail_cleanup/__openerp__.py b/mail_cleanup/__openerp__.py new file mode 100644 index 000000000..88f767f41 --- /dev/null +++ b/mail_cleanup/__openerp__.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Author: Matthieu Dietrich +# Copyright 2015 Camptocamp SA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +{ + 'name': 'Mail cleanup', + 'version': '0.1', + 'category': 'Tools', + 'author': "Camptocamp,Odoo Community Association (OCA)", + 'summary': 'Clean up mails regularly', + 'description': """ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License: AGPL-3 + +Mail cleanup +=========== + +This module allows to mark e-mails older than x days as read +and optionally move them on IMAP servers, just before fetching them. + +Since the main "mail" module does not mark unroutable e-mails as read, +this means that if junk mail arrives in the catch-all address without +any default route, fetching newer e-mails will happen after re-parsing +those unroutable e-mails. + +Configuration +============= + +This module depends on ``mail_environment`` in order to add "expiration dates" +per server. + +Example of a configuration file (add those values to your server):: + + [incoming_mail.openerp_pop_mail1] + cleanup_days = 30 # default value + cleanup_folder = NotParsed # optional parameter + +Known issues / Roadmap +====================== + +* None + +Credits +======= + +Contributors +------------ + +* Matthieu Dietrich + +Maintainer +---------- + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit http://odoo-community.org. +""", + 'license': 'AGPL-3', + 'website': 'http://openerp.camptocamp.com', + 'depends': ['mail_environment'], + 'data': ['mail_view.xml'], + 'installable': True, +} diff --git a/mail_cleanup/mail_cleanup.py b/mail_cleanup/mail_cleanup.py new file mode 100644 index 000000000..7008615de --- /dev/null +++ b/mail_cleanup/mail_cleanup.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Author: Matthieu Dietrich +# Copyright 2015 Camptocamp SA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +import logging +from osv import fields +from osv import osv +from dateutil import relativedelta +from datetime import datetime + +from server_environment import serv_config + +_logger = logging.getLogger(__name__) + + +class FetchmailServer(osv.osv): + """Incoming POP/IMAP mail server account""" + _inherit = 'fetchmail.server' + + def _get_cleanup_conf(self, cursor, uid, ids, name, args, context=None): + """ + Return configuration + """ + res = {} + for fetchmail in self.browse(cursor, uid, ids): + global_section_name = 'incoming_mail' + + # default vals + config_vals = {'cleanup_days': 30, + 'cleanup_folder': False} + if serv_config.has_section(global_section_name): + config_vals.update(serv_config.items(global_section_name)) + + custom_section_name = '.'.join((global_section_name, + fetchmail.name)) + if serv_config.has_section(custom_section_name): + config_vals.update(serv_config.items(custom_section_name)) + + if config_vals.get('cleanup_days'): + config_vals['cleanup_days'] = \ + int(config_vals['cleanup_days']) + if config_vals.get('cleanup_folder'): + config_vals['cleanup_folder'] = \ + config_vals['cleanup_folder'] + res[fetchmail.id] = config_vals + return res + + _columns = { + 'cleanup_days': fields.function( + _get_cleanup_conf, + method=True, + string='Expiration days', + type="integer", + multi='outgoing_mail_config', + help="Number of days before discarding an e-mail"), + 'cleanup_folder': fields.function( + _get_cleanup_conf, + method=True, + string='Expiration folder', + type="char", + multi='outgoing_mail_config', + help="Folder where the discarded e-mail will be moved.") + } + + def _cleanup_fetchmail_server(self, server, imap_server): + count, failed = 0, 0 + expiration_date = (datetime.now() + relativedelta.relativedelta( + days=-(server.cleanup_days))).strftime('%d-%b-%Y') + search_text = '(UNSEEN BEFORE %s)' % expiration_date + imap_server.select() + result, data = imap_server.search(None, search_text) + for num in data[0].split(): + result, data = imap_server.fetch(num, '(RFC822)') + try: + # Mark message as read + imap_server.store(num, '+FLAGS', '\\Seen') + if server.cleanup_folder: + # To move a message, you have to COPY + # then DELETE the message + result = imap_server.copy(num, server.cleanup_folder) + if result[0] == 'OK': + imap_server.store(num, '+FLAGS', '\\Deleted') + imap_server.expunge() + except Exception: + _logger.exception('Failed to cleanup mail from %s server %s.', + server.type, server.name) + failed += 1 + count += 1 + _logger.info("Fetched %d email(s) on %s server %s; " + "%d succeeded, %d failed.", count, server.type, + server.name, (count - failed), failed) + + def fetch_mail(self, cr, uid, ids, context=None): + """ Called before the fetch, in order to clean up + right before retrieving emails. """ + if context is None: + context = {} + context['fetchmail_cron_running'] = True + for server in self.browse(cr, uid, ids, context=context): + _logger.info('start cleaning up emails on %s server %s', + server.type, server.name) + context.update({'fetchmail_server_id': server.id, + 'server_type': server.type}) + imap_server = False + if server.type == 'imap' and server.cleanup_days > 0: + try: + imap_server = server.connect() + self._cleanup_fetchmail_server(server, imap_server) + except Exception: + _logger.exception("General failure when trying to cleanup " + "mail from %s server %s.", + server.type, server.name) + finally: + if imap_server: + imap_server.close() + imap_server.logout() + return super(FetchmailServer, self).fetch_mail(cr, uid, ids, + context=context) diff --git a/mail_cleanup/mail_view.xml b/mail_cleanup/mail_view.xml new file mode 100644 index 000000000..ddcf435d1 --- /dev/null +++ b/mail_cleanup/mail_view.xml @@ -0,0 +1,21 @@ + + + + + + inherit_fetchmail_for cleanup + fetchmail.server + + + + + + + + + + + + + + From 55cb7b5ba3872e4697c9b093fdd33f0f933d5dc1 Mon Sep 17 00:00:00 2001 From: Matthieu Dietrich Date: Tue, 21 Jul 2015 15:12:47 +0200 Subject: [PATCH 2/7] Use correct Model + remove unnecessary conditions/assignments --- mail_cleanup/mail_cleanup.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/mail_cleanup/mail_cleanup.py b/mail_cleanup/mail_cleanup.py index 7008615de..74419719e 100644 --- a/mail_cleanup/mail_cleanup.py +++ b/mail_cleanup/mail_cleanup.py @@ -20,8 +20,7 @@ ############################################################################## import logging -from osv import fields -from osv import osv +from openerp.osv import orm, fields from dateutil import relativedelta from datetime import datetime @@ -30,7 +29,7 @@ from server_environment import serv_config _logger = logging.getLogger(__name__) -class FetchmailServer(osv.osv): +class FetchmailServer(orm.Model): """Incoming POP/IMAP mail server account""" _inherit = 'fetchmail.server' @@ -43,7 +42,7 @@ class FetchmailServer(osv.osv): global_section_name = 'incoming_mail' # default vals - config_vals = {'cleanup_days': 30, + config_vals = {'cleanup_days': "30", 'cleanup_folder': False} if serv_config.has_section(global_section_name): config_vals.update(serv_config.items(global_section_name)) @@ -53,12 +52,8 @@ class FetchmailServer(osv.osv): if serv_config.has_section(custom_section_name): config_vals.update(serv_config.items(custom_section_name)) - if config_vals.get('cleanup_days'): - config_vals['cleanup_days'] = \ - int(config_vals['cleanup_days']) - if config_vals.get('cleanup_folder'): - config_vals['cleanup_folder'] = \ - config_vals['cleanup_folder'] + # convert string value to integer + config_vals['cleanup_days'] = int(config_vals['cleanup_days']) res[fetchmail.id] = config_vals return res From b6058fcd533571aec3597c161d7d2abe9ed3f9a1 Mon Sep 17 00:00:00 2001 From: Matthieu Dietrich Date: Wed, 29 Jul 2015 12:49:08 +0200 Subject: [PATCH 3/7] Add purging mechanism + cleanup is by default inactive --- mail_cleanup/mail_cleanup.py | 51 ++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/mail_cleanup/mail_cleanup.py b/mail_cleanup/mail_cleanup.py index 74419719e..002a05018 100644 --- a/mail_cleanup/mail_cleanup.py +++ b/mail_cleanup/mail_cleanup.py @@ -42,7 +42,8 @@ class FetchmailServer(orm.Model): global_section_name = 'incoming_mail' # default vals - config_vals = {'cleanup_days': "30", + config_vals = {'cleanup_days': False, + 'purge_days': False, 'cleanup_folder': False} if serv_config.has_section(global_section_name): config_vals.update(serv_config.items(global_section_name)) @@ -52,8 +53,11 @@ class FetchmailServer(orm.Model): if serv_config.has_section(custom_section_name): config_vals.update(serv_config.items(custom_section_name)) - # convert string value to integer - config_vals['cleanup_days'] = int(config_vals['cleanup_days']) + # convert string values to integer + if config_vals['cleanup_days']: + config_vals['cleanup_days'] = int(config_vals['cleanup_days']) + if config_vals['purge_days']: + config_vals['purge_days'] = int(config_vals['purge_days']) res[fetchmail.id] = config_vals return res @@ -64,14 +68,21 @@ class FetchmailServer(orm.Model): string='Expiration days', type="integer", multi='outgoing_mail_config', - help="Number of days before discarding an e-mail"), + help="Number of days before marking an e-mail as read"), 'cleanup_folder': fields.function( _get_cleanup_conf, method=True, string='Expiration folder', type="char", multi='outgoing_mail_config', - help="Folder where the discarded e-mail will be moved.") + help="Folder where an e-mail marked as read will be moved."), + 'purge_days': fields.function( + _get_cleanup_conf, + method=True, + string='Deletion days', + type="char", + multi='outgoing_mail_config', + help="Number of days before removing an e-mail"), } def _cleanup_fetchmail_server(self, server, imap_server): @@ -82,7 +93,6 @@ class FetchmailServer(orm.Model): imap_server.select() result, data = imap_server.search(None, search_text) for num in data[0].split(): - result, data = imap_server.fetch(num, '(RFC822)') try: # Mark message as read imap_server.store(num, '+FLAGS', '\\Seen') @@ -98,7 +108,29 @@ class FetchmailServer(orm.Model): server.type, server.name) failed += 1 count += 1 - _logger.info("Fetched %d email(s) on %s server %s; " + _logger.info("Marked %d email(s) as read on %s server %s; " + "%d succeeded, %d failed.", count, server.type, + server.name, (count - failed), failed) + + def _purge_fetchmail_server(self, server, imap_server): + # Purging e-mails older than the purge date, if available + count, failed = 0, 0 + purge_date = (datetime.now() + relativedelta.relativedelta( + days=-(server.purge_days))).strftime('%d-%b-%Y') + search_text = '(BEFORE %s)' % purge_date + imap_server.select() + result, data = imap_server.search(None, search_text) + for num in data[0].split(): + try: + # Delete message + imap_server.store(num, '+FLAGS', '\\Deleted') + imap_server.expunge() + except Exception: + _logger.exception('Failed to remove mail from %s server %s.', + server.type, server.name) + failed += 1 + count += 1 + _logger.info("Removed %d email(s) on %s server %s; " "%d succeeded, %d failed.", count, server.type, server.name, (count - failed), failed) @@ -117,7 +149,10 @@ class FetchmailServer(orm.Model): if server.type == 'imap' and server.cleanup_days > 0: try: imap_server = server.connect() - self._cleanup_fetchmail_server(server, imap_server) + if server.cleanup_days: + self._cleanup_fetchmail_server(server, imap_server) + if server.purge_days: + self._purge_fetchmail_server(server, imap_server) except Exception: _logger.exception("General failure when trying to cleanup " "mail from %s server %s.", From b3b1b2b56e984ae537af4e7d17a65a674ffb5508 Mon Sep 17 00:00:00 2001 From: Matthieu Dietrich Date: Wed, 29 Jul 2015 13:54:44 +0200 Subject: [PATCH 4/7] Add new field + new info in README --- mail_cleanup/README.rst | 10 +++++++--- mail_cleanup/mail_view.xml | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mail_cleanup/README.rst b/mail_cleanup/README.rst index 161ae6eae..041771a88 100644 --- a/mail_cleanup/README.rst +++ b/mail_cleanup/README.rst @@ -4,8 +4,11 @@ Mail cleanup =========== -This module allows to mark e-mails older than x days as read -and optionally move them on IMAP servers, just before fetching them. +This module allows to: +* mark e-mails older than x days as read, +* move those messages in a specific folder, +* remove messages older than x days +on IMAP servers, just before fetching them. Since the main "mail" module does not mark unroutable e-mails as read, this means that if junk mail arrives in the catch-all address without @@ -21,7 +24,8 @@ per server. Example of a configuration file (add those values to your server):: [incoming_mail.openerp_pop_mail1] - cleanup_days = 30 # default value + cleanup_days = False # default value + purge_days = False # default value cleanup_folder = NotParsed # optional parameter Known issues / Roadmap diff --git a/mail_cleanup/mail_view.xml b/mail_cleanup/mail_view.xml index ddcf435d1..26f86b2a6 100644 --- a/mail_cleanup/mail_view.xml +++ b/mail_cleanup/mail_view.xml @@ -11,6 +11,7 @@ + From 694dbb4c942ae16f688ef4d134f35f3b2d09f609 Mon Sep 17 00:00:00 2001 From: Matthieu Dietrich Date: Wed, 29 Jul 2015 16:50:14 +0200 Subject: [PATCH 5/7] Correct type for purge_days + better checks --- mail_cleanup/mail_cleanup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mail_cleanup/mail_cleanup.py b/mail_cleanup/mail_cleanup.py index 002a05018..6f4de5452 100644 --- a/mail_cleanup/mail_cleanup.py +++ b/mail_cleanup/mail_cleanup.py @@ -80,7 +80,7 @@ class FetchmailServer(orm.Model): _get_cleanup_conf, method=True, string='Deletion days', - type="char", + type="integer", multi='outgoing_mail_config', help="Number of days before removing an e-mail"), } @@ -146,12 +146,12 @@ class FetchmailServer(orm.Model): context.update({'fetchmail_server_id': server.id, 'server_type': server.type}) imap_server = False - if server.type == 'imap' and server.cleanup_days > 0: + if server.type == 'imap': try: imap_server = server.connect() - if server.cleanup_days: + if server.cleanup_days > 0: self._cleanup_fetchmail_server(server, imap_server) - if server.purge_days: + if server.purge_days > 0: self._purge_fetchmail_server(server, imap_server) except Exception: _logger.exception("General failure when trying to cleanup " From 7fba5c1307eda7bb2b58072825ff9892c665ae33 Mon Sep 17 00:00:00 2001 From: Matthieu Dietrich Date: Thu, 30 Jul 2015 10:18:25 +0200 Subject: [PATCH 6/7] Place expunge() call after parsing all messages --- mail_cleanup/mail_cleanup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mail_cleanup/mail_cleanup.py b/mail_cleanup/mail_cleanup.py index 6f4de5452..450a9db79 100644 --- a/mail_cleanup/mail_cleanup.py +++ b/mail_cleanup/mail_cleanup.py @@ -102,7 +102,6 @@ class FetchmailServer(orm.Model): result = imap_server.copy(num, server.cleanup_folder) if result[0] == 'OK': imap_server.store(num, '+FLAGS', '\\Deleted') - imap_server.expunge() except Exception: _logger.exception('Failed to cleanup mail from %s server %s.', server.type, server.name) @@ -124,7 +123,6 @@ class FetchmailServer(orm.Model): try: # Delete message imap_server.store(num, '+FLAGS', '\\Deleted') - imap_server.expunge() except Exception: _logger.exception('Failed to remove mail from %s server %s.', server.type, server.name) @@ -153,6 +151,9 @@ class FetchmailServer(orm.Model): self._cleanup_fetchmail_server(server, imap_server) if server.purge_days > 0: self._purge_fetchmail_server(server, imap_server) + # Do the final cleanup: delete all messages + # flagged as deleted + imap_server.expunge() except Exception: _logger.exception("General failure when trying to cleanup " "mail from %s server %s.", From 446cf6f4d4cd32ea546290276cc838ff268c53d3 Mon Sep 17 00:00:00 2001 From: Matthieu Dietrich Date: Mon, 24 Aug 2015 12:20:53 +0200 Subject: [PATCH 7/7] Remove README.rst (not needed in 7.0) --- mail_cleanup/README.rst | 67 ----------------------------------------- 1 file changed, 67 deletions(-) delete mode 100644 mail_cleanup/README.rst diff --git a/mail_cleanup/README.rst b/mail_cleanup/README.rst deleted file mode 100644 index 041771a88..000000000 --- a/mail_cleanup/README.rst +++ /dev/null @@ -1,67 +0,0 @@ -.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg - :alt: License: AGPL-3 - -Mail cleanup -=========== - -This module allows to: -* mark e-mails older than x days as read, -* move those messages in a specific folder, -* remove messages older than x days -on IMAP servers, just before fetching them. - -Since the main "mail" module does not mark unroutable e-mails as read, -this means that if junk mail arrives in the catch-all address without -any default route, fetching newer e-mails will happen after re-parsing -those unroutable e-mails. - -Configuration -============= - -This module depends on ``mail_environment`` in order to add "expiration dates" -per server. - -Example of a configuration file (add those values to your server):: - - [incoming_mail.openerp_pop_mail1] - cleanup_days = False # default value - purge_days = False # default value - cleanup_folder = NotParsed # optional parameter - -Known issues / Roadmap -====================== - -* None - -Bug Tracker -=========== - -Bugs are tracked on `GitHub Issues `_. -In case of trouble, please check there if your issue has already been reported. -If you spotted it first, help us smashing it by providing a detailed and welcomed feedback -`here `_. - - -Credits -======= - -Contributors ------------- - -* Matthieu Dietrich - -Maintainer ----------- - -.. image:: https://odoo-community.org/logo.png - :alt: Odoo Community Association - :target: https://odoo-community.org - -This module is maintained by the OCA. - -OCA, or the Odoo Community Association, is a nonprofit organization whose -mission is to support the collaborative development of Odoo features and -promote its widespread use. - -To contribute to this module, please visit http://odoo-community.org. -