diff --git a/fetchmail_bydate/README.rst b/fetchmail_bydate/README.rst new file mode 100644 index 000000000..635134b97 --- /dev/null +++ b/fetchmail_bydate/README.rst @@ -0,0 +1,87 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :target: http://www.gnu.org/licenses/agpl + :alt: License: AGPL-3 + +================ +Fetchmail Bydate +================ + +This module allows to fetch new emails (using IMAP) received from the last +time they were downloaded and successfully processed, in addition to 'unseen' +status. + +Users with authorization to edit the email server in Odoo can introduce a +new date and time to download from. + +In case of errors found during the processing of an email Odoo will +re-attempt to fetch the emails from the last date and time they were +successfully received and processed. + + + +Configuration +============= + +To enable this, you have to set a 'Last Download Date' in the fetchmail.server +After that, emails with an internal date greater than the saved one will be +downloaded. + + +Usage +===== + +Odoo will attempt to fetch emails starting from the 'Last Download Date' +defined in the email server. If all mails have been processed successfully, +it will update this date with the latest message received. + +System administrators need to be attentive to the Odoo logs, looking for errors +raised during the processing of emails, in order to avoid situations +lots of emails are now being downloaded and reprocessed every time, due to +errors found in a few old emails. + + +.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas + :alt: Try me on Runbot + :target: https://runbot.odoo-community.org/runbot/149/9.0 + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues +`_. In case of trouble, please +check there if your issue has already been reported. If you spotted it first, +help us smash it by providing detailed and welcomed feedback. + +Credits +======= + +Images +------ + +* Odoo Community Association: `Icon `_. + +Contributors +------------ + +* Lorenzo Battistini +* Alessio Gerace +* Jordi Ballester + + +Do not contact contributors directly about support or help with technical issues. + + +Maintainer +---------- + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit https://odoo-community.org. diff --git a/fetchmail_bydate/__init__.py b/fetchmail_bydate/__init__.py index 3ceb4ebf2..ec50cfc0f 100644 --- a/fetchmail_bydate/__init__.py +++ b/fetchmail_bydate/__init__.py @@ -1,25 +1,4 @@ # -*- coding: utf-8 -*- -############################################################################## -# -# Copyright (C) 2015 Innoviu srl (). -# Copyright (C) 2015 Agile Business Group http://www.agilebg.com -# @authors -# Roberto Onnis -# Alessio Gerace -# Lorenzo Battistini -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -############################################################################## +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). -from . import model +from . import models diff --git a/fetchmail_bydate/__openerp__.py b/fetchmail_bydate/__openerp__.py index e3b2cfa34..90d56c33d 100644 --- a/fetchmail_bydate/__openerp__.py +++ b/fetchmail_bydate/__openerp__.py @@ -1,49 +1,29 @@ # -*- coding: utf-8 -*- -############################################################################## -# -# Copyright (C) 2015 Innoviu srl (). -# Copyright (C) 2015 Agile Business Group http://www.agilebg.com -# @authors -# Roberto Onnis -# Alessio Gerace -# Lorenzo Battistini -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -############################################################################## -{ - 'name': "Fetchmail by Date", - "version": "1.0", - 'category': 'Mailing', - 'summary': 'Fetchmail by date and unseen messages', - 'description': """ -This module allows to fetch new emails (using IMAP) using their date, -in addition to 'unseen' status. +# Copyright 2015 Innoviu srl +# Copyright 2015 Agile Business Group +# Copyright 2017 Eficent Business and IT Consulting Services, S.L. +# +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + -To enable this, you have to set a 'Last Download Date' in the fetchmail.server -After that, emails with an internal date greater than the saved one will be -downloaded. -""", - 'author': "Innoviu, Agile Business Group, " +{ + "name": "Fetchmail by Date", + "summary": 'Fetchmail by date and unseen messages', + "version": "9.0.1.0.0", + "category": "Discuss", + "author": "Innoviu, " + "Agile Business Group, " + "Eficent, " "Odoo Community Association (OCA)", - 'website': 'http://www.innoviu.com', - 'license': 'AGPL-3', - 'depends': ['fetchmail', 'mail'], + "website": "https://github.com/OCA/server-tools/tree/9.0", + "license": 'AGPL-3', + "application": False, + "installable": True, + "depends": [ + 'fetchmail', + 'mail', + ], "data": [ - 'view/fetchmail_view.xml', + 'views/fetchmail_view.xml', ], - 'demo': [], - 'test': [], - 'installable': True, - 'auto_install': False, } diff --git a/fetchmail_bydate/model/__init__.py b/fetchmail_bydate/model/__init__.py deleted file mode 100644 index 8cf9e9c84..000000000 --- a/fetchmail_bydate/model/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# Copyright (C) 2015 Innoviu srl (). -# Copyright (C) 2015 Agile Business Group http://www.agilebg.com -# @authors -# Roberto Onnis -# Alessio Gerace -# Lorenzo Battistini -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -############################################################################## - -from . import fetchmail diff --git a/fetchmail_bydate/model/fetchmail.py b/fetchmail_bydate/model/fetchmail.py deleted file mode 100644 index a17c65dfb..000000000 --- a/fetchmail_bydate/model/fetchmail.py +++ /dev/null @@ -1,152 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# Copyright (C) 2015 Innoviu srl (). -# Copyright (C) 2015 Agile Business Group http://www.agilebg.com -# @authors -# Roberto Onnis -# Alessio Gerace -# Lorenzo Battistini -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -############################################################################## -from openerp.osv import fields, orm -import logging -import imaplib -from datetime import datetime -import time - - -_logger = logging.getLogger(__name__) - - -class FetchmailServer(orm.Model): - - _inherit = "fetchmail.server" - - _columns = { - 'last_internal_date': fields.datetime( - 'Last Download Date', - help="Remote emails with a date greater than this will be " - "downloaded. Only available with IMAP"), - } - - def _fetch_from_date_imap(self, cr, uid, - server, imap_server, - mail_thread, action_pool, - count, failed, - context=None): - messages = [] - date_uids = {} - last_date = False - last_internal_date = datetime.strptime( - server.last_internal_date, "%Y-%m-%d %H:%M:%S") - search_status, uids = imap_server.search( - None, - 'SINCE', '%s' % last_internal_date.strftime('%d-%b-%Y') - ) - new_uids = uids[0].split() - for new_uid in new_uids: - fetch_status, date = imap_server.fetch( - int(new_uid), - 'INTERNALDATE' - ) - internaldate = imaplib.Internaldate2tuple(date[0]) - internaldate_msg = datetime.fromtimestamp( - time.mktime(internaldate) - ) - if internaldate_msg > last_internal_date: - messages.append(new_uid) - date_uids[new_uid] = internaldate_msg - for num in messages: - # SEARCH command *always* returns at least the most - # recent message, even if it has already been synced - res_id = None - result, data = imap_server.fetch(num, '(RFC822)') - if data and data[0]: - try: - res_id = mail_thread.message_process( - cr, uid, - server.object_id.model, - data[0][1], - save_original=server.original, - strip_attachments=(not server.attach), - context=context) - except Exception: - _logger.exception( - 'Failed to process mail \ - from %s server %s.', - server.type, - server.name) - failed += 1 - if res_id and server.action_id: - action_pool.run( - cr, uid, [server.action_id.id], - { - 'active_id': res_id, - 'active_ids': [res_id], - 'active_model': context.get( - "thread_model", - server.object_id.model) - }, context=context) - imap_server.store(num, '+FLAGS', '\\Seen') - cr.commit() - count += 1 - last_date = not failed and date_uids[num] or False - return count, failed, last_date - - def fetch_mail(self, cr, uid, ids, context=None): - if context is None: - context = {} - context['fetchmail_cron_running'] = True - mail_thread = self.pool.get('mail.thread') - action_pool = self.pool.get('ir.actions.server') - for server in self.browse(cr, uid, ids, context=context): - if server.type == 'imap' and server.last_internal_date: - _logger.info( - 'start checking for new emails by date on %s server %s', - server.type, server.name) - context.update({'fetchmail_server_id': server.id, - 'server_type': server.type}) - count, failed = 0, 0 - last_date = False - imap_server = False - try: - imap_server = server.connect() - imap_server.select() - count, failed, last_date = self._fetch_from_date_imap( - cr, uid, server, imap_server, mail_thread, - action_pool, count, failed, context=context - ) - except Exception: - _logger.exception( - "General failure when trying to fetch mail \ - from %s server %s.", - server.type, - server.name - ) - finally: - if imap_server: - imap_server.close() - imap_server.logout() - if last_date: - _logger.info( - "Fetched %d email(s) on %s server %s; \ - %d succeeded, %d failed.", count, - server.type, server.name, - (count - failed), failed) - vals = {'last_internal_date': last_date} - server.write(vals) - return super(FetchmailServer, self).fetch_mail( - cr, uid, ids, context=context) diff --git a/fetchmail_bydate/models/__init__.py b/fetchmail_bydate/models/__init__.py new file mode 100644 index 000000000..66aa6eec2 --- /dev/null +++ b/fetchmail_bydate/models/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from . import fetchmail diff --git a/fetchmail_bydate/models/fetchmail.py b/fetchmail_bydate/models/fetchmail.py new file mode 100644 index 000000000..3caba14a9 --- /dev/null +++ b/fetchmail_bydate/models/fetchmail.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Innoviu srl +# Copyright 2015 Agile Business Group +# Copyright 2017 Eficent Business and IT Consulting Services, S.L. +# +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import logging +import imaplib +from datetime import datetime +import time +from openerp import api, fields, models + +_logger = logging.getLogger(__name__) + + +class FetchmailServer(models.Model): + + _inherit = "fetchmail.server" + + last_internal_date = fields.Datetime( + 'Last Download Date', + help="Remote emails with a date greater than this will be " + "downloaded. Only available with IMAP") + + @api.model + def _fetch_from_date_imap(self, imap_server, count, failed): + MailThread = self.env['mail.thread'] + messages = [] + date_uids = {} + last_date = False + last_internal_date = datetime.strptime(self.last_internal_date, + "%Y-%m-%d %H:%M:%S") + search_status, uids = imap_server.search( + None, + 'SINCE', '%s' % last_internal_date.strftime('%d-%b-%Y') + ) + new_uids = uids[0].split() + for new_uid in new_uids: + fetch_status, date = imap_server.fetch( + int(new_uid), + 'INTERNALDATE' + ) + internaldate = imaplib.Internaldate2tuple(date[0]) + internaldate_msg = datetime.fromtimestamp( + time.mktime(internaldate) + ) + if internaldate_msg > last_internal_date: + messages.append(new_uid) + date_uids[new_uid] = internaldate_msg + for num in messages: + # SEARCH command *always* returns at least the most + # recent message, even if it has already been synced + res_id = None + result, data = imap_server.fetch(num, '(RFC822)') + if data and data[0]: + try: + res_id = MailThread.message_process( + self.object_id.model, + data[0][1], + save_original=self.original, + strip_attachments=(not self.attach)) + except Exception: + _logger.exception( + 'Failed to process mail \ + from %s server %s.', + self.type, + self.name) + failed += 1 + if res_id and self.action_id: + self.action_id.run({ + 'active_id': res_id, + 'active_ids': [res_id], + 'active_model': self.env.context.get( + "thread_model", self.object_id.model) + }) + imap_server.store(num, '+FLAGS', '\\Seen') + self._cr.commit() + count += 1 + last_date = not failed and date_uids[num] or False + return count, failed, last_date + + @api.multi + def fetch_mail(self): + context = self.env.context.copy() + context['fetchmail_cron_running'] = True + for server in self: + if server.type == 'imap' and server.last_internal_date: + _logger.info( + 'start checking for new emails, starting from %s on %s ' + 'server %s', + server.last_internal_date, server.type, server.name) + context.update({'fetchmail_server_id': server.id, + 'server_type': server.type}) + count, failed = 0, 0 + last_date = False + imap_server = False + try: + imap_server = server.connect() + imap_server.select() + count, failed, last_date = server._fetch_from_date_imap( + imap_server, count, failed) + except Exception: + _logger.exception( + "General failure when trying to fetch mail by date \ + from %s server %s.", + server.type, + server.name + ) + finally: + if imap_server: + imap_server.close() + imap_server.logout() + if last_date: + _logger.info( + "Fetched %d email(s) on %s server %s, starting from " + "%s; %d succeeded, %d failed.", count, + server.type, server.name, last_date, + (count - failed), failed) + vals = {'last_internal_date': last_date} + server.write(vals) + return super(FetchmailServer, self).fetch_mail() diff --git a/fetchmail_bydate/view/fetchmail_view.xml b/fetchmail_bydate/view/fetchmail_view.xml deleted file mode 100644 index 6a027ace9..000000000 --- a/fetchmail_bydate/view/fetchmail_view.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - fetchmail.bydate.form - fetchmail.server - - - - - - - - - - - diff --git a/fetchmail_bydate/views/fetchmail_view.xml b/fetchmail_bydate/views/fetchmail_view.xml new file mode 100644 index 000000000..130376119 --- /dev/null +++ b/fetchmail_bydate/views/fetchmail_view.xml @@ -0,0 +1,23 @@ + + + + + + + fetchmail.bydate.form + fetchmail.server + + + + + + + + + +