9.0 mail cleanup (#410)

* Add new module "mail_cleanup" in order to move/mark as read old messages

* Use correct Model + remove unnecessary conditions/assignments

* Add purging mechanism + cleanup is by default inactive

* Add new field + new info in README

* Correct type for purge_days + better checks

* Place expunge() call after parsing all messages

* Migration to 9.0 of mail_cleanup

* Fix README.rst

* Add __init__.py

* Fix error with multi mail servers

* Correct syntax for methods + use datetime.date.today

mail_cleanup/README.rst

@@ -0,0 +1,69 @@
+.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+   :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+   :alt: License: AGPL-3
+
+============
+Mail cleanup
+===========
+
+This module allows to:
+* mark e-mails older than x days as read,
+* move those messages in a specific folder,
+* remove messages older than x days
+on IMAP servers, just before fetching them.
+
+Since the main "mail" module does not mark unroutable e-mails as read,
+this means that if junk mail arrives in the catch-all address without
+any default route, fetching newer e-mails will happen after re-parsing
+those unroutable e-mails.
+
+Configuration
+=============
+
+This module depends on ``mail_environment`` in order to add "expiration dates"
+per server.
+
+Example of a configuration file (add those values to your server)::
+
+ [incoming_mail.openerp_imap_mail1]
+ cleanup_days = False # default value
+ purge_days = False # default value
+ cleanup_folder = NotParsed # optional parameter
+
+Known issues / Roadmap
+======================
+
+* None
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/server-tools/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us smashing it by providing a detailed and welcomed feedback
+`here <https://github.com/OCA/server-tools/issues/new?body=module:%20mail_cleanup%0Aversion:%209.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+
+
+Credits
+=======
+
+Contributors
+------------
+
+* Matthieu Dietrich <matthieu.dietrich@camptocamp.com>
+
+Maintainer
+----------
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+This module is maintained by the OCA.
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+To contribute to this module, please visit https://odoo-community.org.
+

mail_cleanup/__init__.py

@@ -0,0 +1 @@
+from . import models

mail_cleanup/__openerp__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# © 2015-2016 Matthieu Dietrich (Camptocamp SA)
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+{
+    'name': 'Mail cleanup',
+    'version': '9.0.1.0.0',
+    'category': 'Tools',
+    'summary': 'Mark as read or delete mails after a set time',
+    'author': "Camptocamp,Odoo Community Association (OCA)",
+    'license': 'AGPL-3',
+    'website': 'https://odoo-community.org',
+    'depends': ['mail_environment'],
+    'data': ['views/mail_view.xml'],
+    'installable': True,
+}

mail_cleanup/models/__init__.py

@@ -0,0 +1 @@
+from . import mail_cleanup

mail_cleanup/models/mail_cleanup.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+# © 2015-2016 Matthieu Dietrich (Camptocamp SA)
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+import logging
+import datetime
+from openerp import api, fields, models
+from dateutil.relativedelta import relativedelta
+
+from openerp.addons.server_environment import serv_config
+
+_logger = logging.getLogger(__name__)
+
+
+class FetchmailServer(models.Model):
+    """Incoming POP/IMAP mail server account"""
+    _inherit = 'fetchmail.server'
+
+    cleanup_days = fields.Integer(
+        compute='_get_cleanup_conf',
+        string='Expiration days',
+        help="Number of days before marking an e-mail as read")
+
+    cleanup_folder = fields.Char(
+        compute='_get_cleanup_conf',
+        string='Expiration folder',
+        help="Folder where an e-mail marked as read will be moved.")
+
+    purge_days = fields.Integer(
+        compute='_get_cleanup_conf',
+        string='Deletion days',
+        help="Number of days before removing an e-mail")
+
+    @api.multi
+    def _get_cleanup_conf(self):
+        """
+        Return configuration
+        """
+        for fetchmail in self:
+            global_section_name = 'incoming_mail'
+
+            # default vals
+            config_vals = {'cleanup_days': False,
+                           'purge_days': False,
+                           'cleanup_folder': False}
+            if serv_config.has_section(global_section_name):
+                config_vals.update(serv_config.items(global_section_name))
+
+            custom_section_name = '.'.join((global_section_name,
+                                            fetchmail.name))
+            if serv_config.has_section(custom_section_name):
+                config_vals.update(serv_config.items(custom_section_name))
+
+            # convert string values to integer
+            if config_vals['cleanup_days']:
+                config_vals['cleanup_days'] = int(config_vals['cleanup_days'])
+            if config_vals['purge_days']:
+                config_vals['purge_days'] = int(config_vals['purge_days'])
+
+            for field in ['cleanup_days', 'purge_days', 'cleanup_folder']:
+                fetchmail[field] = config_vals[field]
+
+    def _cleanup_fetchmail_server(self, server, imap_server):
+        count, failed = 0, 0
+        expiration_date = datetime.date.today()
+        expiration_date -= relativedelta(days=server.cleanup_days)
+        search_text = expiration_date.strftime('(UNSEEN BEFORE %d-%b-%Y)')
+        imap_server.select()
+        result, data = imap_server.search(None, search_text)
+        for num in data[0].split():
+            try:
+                # Mark message as read
+                imap_server.store(num, '+FLAGS', '\\Seen')
+                if server.cleanup_folder:
+                    # To move a message, you have to COPY
+                    # then DELETE the message
+                    result = imap_server.copy(num, server.cleanup_folder)
+                    if result[0] == 'OK':
+                        imap_server.store(num, '+FLAGS', '\\Deleted')
+            except Exception:
+                _logger.exception('Failed to cleanup mail from %s server %s.',
+                                  server.type, server.name)
+                failed += 1
+            count += 1
+        _logger.info("Marked %d email(s) as read on %s server %s; "
+                     "%d succeeded, %d failed.", count, server.type,
+                     server.name, (count - failed), failed)
+
+    def _purge_fetchmail_server(self, server, imap_server):
+        # Purging e-mails older than the purge date, if available
+        count, failed = 0, 0
+        purge_date = datetime.date.today()
+        purge_date -= relativedelta(days=server.purge_days)
+        search_text = purge_date.strftime('(BEFORE %d-%b-%Y)')
+        imap_server.select()
+        result, data = imap_server.search(None, search_text)
+        for num in data[0].split():
+            try:
+                # Delete message
+                imap_server.store(num, '+FLAGS', '\\Deleted')
+            except Exception:
+                _logger.exception('Failed to remove mail from %s server %s.',
+                                  server.type, server.name)
+                failed += 1
+            count += 1
+        _logger.info("Removed %d email(s) on %s server %s; "
+                     "%d succeeded, %d failed.", count, server.type,
+                     server.name, (count - failed), failed)
+
+    @api.multi
+    def fetch_mail(self):
+        """ Called before the fetch, in order to clean up
+             right before retrieving emails. """
+        context = self.env.context.copy()
+        context['fetchmail_cron_running'] = True
+        for server in self:
+            _logger.info('start cleaning up emails on %s server %s',
+                         server.type, server.name)
+            context.update({'fetchmail_server_id': server.id,
+                            'server_type': server.type})
+            imap_server = False
+            if server.type == 'imap':
+                try:
+                    imap_server = server.connect()
+                    if server.cleanup_days > 0:
+                        self._cleanup_fetchmail_server(server, imap_server)
+                    if server.purge_days > 0:
+                        self._purge_fetchmail_server(server, imap_server)
+                    # Do the final cleanup: delete all messages
+                    # flagged as deleted
+                    imap_server.expunge()
+                except Exception:
+                    _logger.exception("General failure when trying to cleanup "
+                                      "mail from %s server %s.",
+                                      server.type, server.name)
+                finally:
+                    if imap_server:
+                        imap_server.close()
+                        imap_server.logout()
+        return super(FetchmailServer, self).fetch_mail()

mail_cleanup/views/mail_view.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+    <record model="ir.ui.view" id="inherit_fetchmail_cleanup">
+        <!-- must be unique in this module. -->
+        <field name="name">inherit_fetchmail_for cleanup</field>
+        <field name="model">fetchmail.server</field>
+        <!--parent python entity -->
+        <field name="inherit_id" ref="mail_environment.inherit_fetchmail"/>
+        <!-- modulename.view -->
+        <field name="arch" type="xml">
+            <field name="is_ssl" position="after">
+                <field name="cleanup_days"/>
+                <field name="purge_days"/>
+            </field>
+            <field name="password" position="after">
+                <field name="cleanup_folder"/>
+            </field>
+        </field>
+    </record>
+</odoo>