From 0c2aae5a2fcd03836dfbc1839a5296ad338abb9e Mon Sep 17 00:00:00 2001 From: archetipo Date: Fri, 6 Mar 2015 09:29:24 +0100 Subject: [PATCH 1/7] [IMP] first commit --- fetchmail_bydate/__init__.py | 25 ++ fetchmail_bydate/__openerp__.py | 44 ++++ fetchmail_bydate/model/__init__.py | 25 ++ fetchmail_bydate/model/fetchmail.py | 276 +++++++++++++++++++++++ fetchmail_bydate/view/fetchmail_view.xml | 18 ++ 5 files changed, 388 insertions(+) create mode 100644 fetchmail_bydate/__init__.py create mode 100644 fetchmail_bydate/__openerp__.py create mode 100644 fetchmail_bydate/model/__init__.py create mode 100644 fetchmail_bydate/model/fetchmail.py create mode 100644 fetchmail_bydate/view/fetchmail_view.xml diff --git a/fetchmail_bydate/__init__.py b/fetchmail_bydate/__init__.py new file mode 100644 index 000000000..3ceb4ebf2 --- /dev/null +++ b/fetchmail_bydate/__init__.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (C) 2015 Innoviu srl (). +# Copyright (C) 2015 Agile Business Group http://www.agilebg.com +# @authors +# Roberto Onnis +# Alessio Gerace +# Lorenzo Battistini +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +############################################################################## + +from . import model diff --git a/fetchmail_bydate/__openerp__.py b/fetchmail_bydate/__openerp__.py new file mode 100644 index 000000000..46a866e6e --- /dev/null +++ b/fetchmail_bydate/__openerp__.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (C) 2015 Innoviu srl (). +# Copyright (C) 2015 Agile Business Group http://www.agilebg.com +# @authors +# Roberto Onnis +# Alessio Gerace +# Lorenzo Battistini +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +############################################################################## +{ + 'name': "Fetchmail by Date", + "version": "1.0", + 'category': 'Mailing', + 'summary': 'This module allows to fetch email by last message internal date and unseen messages.', + 'description': """ + This module allows to fetch email by last message internal date and unseen messages. + """, + 'author': "Innoviu srl, Agile Business Group, Odoo Community Association (OCA)", + 'website': 'http://www.innoviu.it, http://www.agilebg.com', + 'license': 'AGPL-3', + 'depends': ['fetchmail', 'mail'], + "data": [ + 'view/fetchmail_view.xml', + ], + 'demo': [], + 'test': [], + 'installable': True, + 'application': True, + 'auto_install': False, +} diff --git a/fetchmail_bydate/model/__init__.py b/fetchmail_bydate/model/__init__.py new file mode 100644 index 000000000..dd501ffb4 --- /dev/null +++ b/fetchmail_bydate/model/__init__.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (C) 2015 Innoviu srl (). +# Copyright (C) 2015 Agile Business Group http://www.agilebg.com +# @authors +# Roberto Onnis +# Alessio Gerace +# Lorenzo Battistini +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +############################################################################## + +from . import fetchmail diff --git a/fetchmail_bydate/model/fetchmail.py b/fetchmail_bydate/model/fetchmail.py new file mode 100644 index 000000000..863cc3435 --- /dev/null +++ b/fetchmail_bydate/model/fetchmail.py @@ -0,0 +1,276 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (C) 2015 Innoviu srl (). +# Copyright (C) 2015 Agile Business Group http://www.agilebg.com +# @authors +# Roberto Onnis +# Alessio Gerace +# Lorenzo Battistini +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +############################################################################## +from openerp.osv import fields, orm +from openerp.tools import ( + DEFAULT_SERVER_DATETIME_FORMAT as DSDTF) +import logging +import imaplib +from datetime import datetime +import time +import calendar + + +_logger = logging.getLogger(__name__) + + +class FetchmailServer(orm.Model): + + _inherit = "fetchmail.server" + + _columns = { + 'last_internal_date': fields.datetime('Last Internal Date'), + } + + def _fetch_from_data_imap(self, cr, uid, + server, imap_server, + mail_thread, action_pool, + count, failed, + context=None): + messages = [] + date_uids = {} + last_date = False + last_internal_date = datetime.strptime( + server.last_internal_date, DSDTF) + timestamp1 = calendar.timegm( + last_internal_date.timetuple()) + intDate = imaplib.Time2Internaldate(timestamp1) + search_status, uids = imap_server.search( + None, + 'SINCE', + '%s' % intDate + ) + new_uids = uids[0].split() + for new_uid in new_uids: + fetch_status, data = imap_server.fetch( + int(new_uid), + 'INTERNALDATE' + ) + internaldate = imaplib.Internaldate2tuple(data[0]) + internaldate_msg = datetime.fromtimestamp( + time.mktime(internaldate) + ) + if internaldate_msg > last_internal_date: + messages.append(new_uid) + date_uids[new_uid] = internaldate_msg + for num in messages: + # SEARCH command *always* returns at least the most + # recent message, even if it has already been synced + res_id = None + result, data = imap_server.uid('fetch', num, + '(RFC822)') + imap_server.store(num, '-FLAGS', '\\Seen') + try: + res_id = mail_thread.message_process( + cr, uid, + server.object_id.model, + data[0][1], + save_original=server.original, + strip_attachments=(not server.attach), + context=context) + except Exception: + _logger.exception( + 'Failed to process mail \ + from %s server %s.', + server.type, + server.name) + failed += 1 + if res_id and server.action_id: + action_pool.run(cr, uid, [server.action_id.id], + {'active_id': res_id, + 'active_ids': [res_id], + 'active_model': context.get( + "thread_model", + server.object_id.model)} + ) + imap_server.store(num, '+FLAGS', '\\Seen') + cr.commit() + count += 1 + last_date = not failed and date_uids[num] or False + return count, failed, last_date + + def _fetch_unread_imap(self, cr, uid, + server, imap_server, + mail_thread, action_pool, + count, failed, + context=None): + result, data = imap_server.search(None, '(UNSEEN)') + for num in data[0].split(): + res_id = None + result, data = imap_server.fetch(num, '(RFC822)') + imap_server.store(num, '-FLAGS', '\\Seen') + try: + res_id = mail_thread.message_process( + cr, uid, server.object_id.model, + data[0][1], + save_original=server.original, + strip_attachments=(not server.attach), + context=context) + except Exception: + _logger.exception( + 'Failed to process mail \ + from %s server %s.', + server.type, + server.name) + failed += 1 + if res_id and server.action_id: + action_pool.run(cr, uid, + [server.action_id.id], + {'active_id': res_id, + 'active_ids': [res_id], + 'active_model': context.get( + "thread_model", + server.object_id.model)} + ) + imap_server.store(num, '+FLAGS', '\\Seen') + cr.commit() + count += 1 + return count, failed + + def _fetch_unread_pop(self, cr, uid, + server, mail_thread, + failed, action_pool, + context=None): + try: + pop_server = server.connect() + (numMsgs, totalSize) = pop_server.stat() + pop_server.list() + for num in range(1, numMsgs + 1): + (header, msges, octets) = pop_server.retr(num) + msg = '\n'.join(msges) + res_id = None + try: + res_id = mail_thread.message_process( + cr, uid, server.object_id.model, + msg, + save_original=server.original, + strip_attachments=(not server.attach), + context=context) + except Exception: + _logger.exception( + 'Failed to process mail \ + from %s server %s.', + server.type, + server.name) + failed += 1 + if res_id and server.action_id: + action_pool.run(cr, uid, [server.action_id.id], + {'active_id': res_id, + 'active_ids': [res_id], + 'active_model': context.get( + "thread_model", + server.object_id.model)} + ) + pop_server.dele(num) + cr.commit() + _logger.info( + "Fetched %d email(s) on %s server %s; \ + %d succeeded, %d failed.", + numMsgs, + server.type, + server.name, + (numMsgs - failed), + failed) + except Exception: + _logger.exception( + "General failure when trying to fetch \ + mail from %s server %s.", + server.type, + server.name) + finally: + if pop_server: + pop_server.quit() + + def fetch_mail(self, cr, uid, ids, context=None): + """WARNING: meant for cron usage only - + will commit() after each email! + """ + if context is None: + context = {} + context['fetchmail_cron_running'] = True + mail_thread = self.pool.get('mail.thread') + action_pool = self.pool.get('ir.actions.server') + for server in self.browse(cr, uid, ids, context=context): + _logger.info('start checking for new emails on %s server %s', + server.type, server.name) + context.update({'fetchmail_server_id': server.id, + 'server_type': server.type}) + count, failed = 0, 0 + last_date = False + imap_server = False + if server.type == 'imap': + try: + imap_server = server.connect() + imap_server.select() + if server.last_internal_date: + count, failed, last_date = self._fetch_from_data_imap( + cr, + uid, + server, + imap_server, + mail_thread, + action_pool, + count, + failed, + context=context) + count, failed = self._fetch_unread_imap( + cr, + uid, + server, + imap_server, + mail_thread, + action_pool, + count, + failed, + context=context) + _logger.info( + "Fetched %d email(s) on %s server %s; \ + %d succeeded, %d failed.", + count, + server.type, + server.name, + (count - failed), + failed) + except Exception: + _logger.exception( + "General failure when trying to fetch mail \ + from %s server %s.", + server.type, + server.name + ) + finally: + if imap_server: + imap_server.close() + imap_server.logout() + elif server.type == 'pop': + self._fetch_unread_pop(cr, uid, + server, mail_thread, + failed, action_pool, + context=context) + vals = {'date': + time.strftime(DSDTF) + } + if last_date: + vals['last_internal_date'] = last_date + server.write(vals) + return True diff --git a/fetchmail_bydate/view/fetchmail_view.xml b/fetchmail_bydate/view/fetchmail_view.xml new file mode 100644 index 000000000..6a027ace9 --- /dev/null +++ b/fetchmail_bydate/view/fetchmail_view.xml @@ -0,0 +1,18 @@ + + + + + + fetchmail.bydate.form + fetchmail.server + + + + + + + + + + + From 5b2db7504b5cc739b4261350dc8dfc9b5b9ef26a Mon Sep 17 00:00:00 2001 From: archetipo Date: Fri, 6 Mar 2015 18:05:16 +0100 Subject: [PATCH 2/7] [IMP] override fetch_mail and if is defined last_internal_date fetch all message from this date to today and call message_process, only for imap server type --- fetchmail_bydate/model/fetchmail.py | 189 +++++----------------------- 1 file changed, 32 insertions(+), 157 deletions(-) diff --git a/fetchmail_bydate/model/fetchmail.py b/fetchmail_bydate/model/fetchmail.py index 863cc3435..7e8768fdd 100644 --- a/fetchmail_bydate/model/fetchmail.py +++ b/fetchmail_bydate/model/fetchmail.py @@ -22,13 +22,10 @@ # along with this program. If not, see . ############################################################################## from openerp.osv import fields, orm -from openerp.tools import ( - DEFAULT_SERVER_DATETIME_FORMAT as DSDTF) import logging import imaplib from datetime import datetime import time -import calendar _logger = logging.getLogger(__name__) @@ -51,14 +48,12 @@ class FetchmailServer(orm.Model): date_uids = {} last_date = False last_internal_date = datetime.strptime( - server.last_internal_date, DSDTF) - timestamp1 = calendar.timegm( - last_internal_date.timetuple()) - intDate = imaplib.Time2Internaldate(timestamp1) + server.last_internal_date, "%Y-%m-%d %H:%M:%S") + #~ timestamp1 = time.mktime(last_internal_date.timetuple()) + #~ intDate = imaplib.Time2Internaldate(timestamp1) search_status, uids = imap_server.search( None, - 'SINCE', - '%s' % intDate + 'SINCE', '%s' % last_internal_date.strftime('%d-%b-%Y') ) new_uids = uids[0].split() for new_uid in new_uids: @@ -76,93 +71,15 @@ class FetchmailServer(orm.Model): for num in messages: # SEARCH command *always* returns at least the most # recent message, even if it has already been synced - res_id = None - result, data = imap_server.uid('fetch', num, - '(RFC822)') - imap_server.store(num, '-FLAGS', '\\Seen') - try: - res_id = mail_thread.message_process( - cr, uid, - server.object_id.model, - data[0][1], - save_original=server.original, - strip_attachments=(not server.attach), - context=context) - except Exception: - _logger.exception( - 'Failed to process mail \ - from %s server %s.', - server.type, - server.name) - failed += 1 - if res_id and server.action_id: - action_pool.run(cr, uid, [server.action_id.id], - {'active_id': res_id, - 'active_ids': [res_id], - 'active_model': context.get( - "thread_model", - server.object_id.model)} - ) - imap_server.store(num, '+FLAGS', '\\Seen') - cr.commit() - count += 1 - last_date = not failed and date_uids[num] or False - return count, failed, last_date - - def _fetch_unread_imap(self, cr, uid, - server, imap_server, - mail_thread, action_pool, - count, failed, - context=None): - result, data = imap_server.search(None, '(UNSEEN)') - for num in data[0].split(): res_id = None result, data = imap_server.fetch(num, '(RFC822)') imap_server.store(num, '-FLAGS', '\\Seen') - try: - res_id = mail_thread.message_process( - cr, uid, server.object_id.model, - data[0][1], - save_original=server.original, - strip_attachments=(not server.attach), - context=context) - except Exception: - _logger.exception( - 'Failed to process mail \ - from %s server %s.', - server.type, - server.name) - failed += 1 - if res_id and server.action_id: - action_pool.run(cr, uid, - [server.action_id.id], - {'active_id': res_id, - 'active_ids': [res_id], - 'active_model': context.get( - "thread_model", - server.object_id.model)} - ) - imap_server.store(num, '+FLAGS', '\\Seen') - cr.commit() - count += 1 - return count, failed - - def _fetch_unread_pop(self, cr, uid, - server, mail_thread, - failed, action_pool, - context=None): - try: - pop_server = server.connect() - (numMsgs, totalSize) = pop_server.stat() - pop_server.list() - for num in range(1, numMsgs + 1): - (header, msges, octets) = pop_server.retr(num) - msg = '\n'.join(msges) - res_id = None + if data and data[0]: try: res_id = mail_thread.message_process( - cr, uid, server.object_id.model, - msg, + cr, uid, + server.object_id.model, + data[0][1], save_original=server.original, strip_attachments=(not server.attach), context=context) @@ -174,32 +91,20 @@ class FetchmailServer(orm.Model): server.name) failed += 1 if res_id and server.action_id: - action_pool.run(cr, uid, [server.action_id.id], - {'active_id': res_id, - 'active_ids': [res_id], - 'active_model': context.get( - "thread_model", - server.object_id.model)} - ) - pop_server.dele(num) - cr.commit() - _logger.info( - "Fetched %d email(s) on %s server %s; \ - %d succeeded, %d failed.", - numMsgs, - server.type, - server.name, - (numMsgs - failed), - failed) - except Exception: - _logger.exception( - "General failure when trying to fetch \ - mail from %s server %s.", - server.type, - server.name) - finally: - if pop_server: - pop_server.quit() + action_pool.run( + cr, uid, [server.action_id.id], + { + 'active_id': res_id, + 'active_ids': [res_id], + 'active_model': context.get( + "thread_model", + server.object_id.model) + }, context=context) + imap_server.store(num, '+FLAGS', '\\Seen') + cr.commit() + count += 1 + last_date = not failed and date_uids[num] or False + return count, failed, last_date def fetch_mail(self, cr, uid, ids, context=None): """WARNING: meant for cron usage only - @@ -211,8 +116,9 @@ class FetchmailServer(orm.Model): mail_thread = self.pool.get('mail.thread') action_pool = self.pool.get('ir.actions.server') for server in self.browse(cr, uid, ids, context=context): - _logger.info('start checking for new emails on %s server %s', - server.type, server.name) + _logger.info( + 'start checking for new emails by date on %s server %s', + server.type, server.name) context.update({'fetchmail_server_id': server.id, 'server_type': server.type}) count, failed = 0, 0 @@ -224,33 +130,9 @@ class FetchmailServer(orm.Model): imap_server.select() if server.last_internal_date: count, failed, last_date = self._fetch_from_data_imap( - cr, - uid, - server, - imap_server, - mail_thread, - action_pool, - count, - failed, - context=context) - count, failed = self._fetch_unread_imap( - cr, - uid, - server, - imap_server, - mail_thread, - action_pool, - count, - failed, - context=context) - _logger.info( - "Fetched %d email(s) on %s server %s; \ - %d succeeded, %d failed.", - count, - server.type, - server.name, - (count - failed), - failed) + cr, uid, server, imap_server, mail_thread, + action_pool, count, failed, context=context + ) except Exception: _logger.exception( "General failure when trying to fetch mail \ @@ -262,15 +144,8 @@ class FetchmailServer(orm.Model): if imap_server: imap_server.close() imap_server.logout() - elif server.type == 'pop': - self._fetch_unread_pop(cr, uid, - server, mail_thread, - failed, action_pool, - context=context) - vals = {'date': - time.strftime(DSDTF) - } if last_date: - vals['last_internal_date'] = last_date - server.write(vals) - return True + vals = {'last_internal_date': last_date} + server.write(vals) + return super(FetchmailServer, self).fetch_mail( + cr, uid, ids, context=context) From 5ef9eb8825fb7b9831f3cda5d01808abe1fa5855 Mon Sep 17 00:00:00 2001 From: archetipo Date: Mon, 9 Mar 2015 14:31:41 +0100 Subject: [PATCH 3/7] [FIX] module is not an App [FIX] check last_internal_date before connect server [FIX] display count an field message retrieved --- fetchmail_bydate/__openerp__.py | 3 +-- fetchmail_bydate/model/fetchmail.py | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/fetchmail_bydate/__openerp__.py b/fetchmail_bydate/__openerp__.py index 46a866e6e..5b40292c7 100644 --- a/fetchmail_bydate/__openerp__.py +++ b/fetchmail_bydate/__openerp__.py @@ -30,7 +30,7 @@ This module allows to fetch email by last message internal date and unseen messages. """, 'author': "Innoviu srl, Agile Business Group, Odoo Community Association (OCA)", - 'website': 'http://www.innoviu.it, http://www.agilebg.com', + 'website': 'http://www.innoviu.it http://www.agilebg.com', 'license': 'AGPL-3', 'depends': ['fetchmail', 'mail'], "data": [ @@ -39,6 +39,5 @@ 'demo': [], 'test': [], 'installable': True, - 'application': True, 'auto_install': False, } diff --git a/fetchmail_bydate/model/fetchmail.py b/fetchmail_bydate/model/fetchmail.py index 7e8768fdd..acc417711 100644 --- a/fetchmail_bydate/model/fetchmail.py +++ b/fetchmail_bydate/model/fetchmail.py @@ -124,15 +124,14 @@ class FetchmailServer(orm.Model): count, failed = 0, 0 last_date = False imap_server = False - if server.type == 'imap': + if server.type == 'imap' and server.last_internal_date: try: imap_server = server.connect() imap_server.select() - if server.last_internal_date: - count, failed, last_date = self._fetch_from_data_imap( - cr, uid, server, imap_server, mail_thread, - action_pool, count, failed, context=context - ) + count, failed, last_date = self._fetch_from_data_imap( + cr, uid, server, imap_server, mail_thread, + action_pool, count, failed, context=context + ) except Exception: _logger.exception( "General failure when trying to fetch mail \ @@ -144,8 +143,13 @@ class FetchmailServer(orm.Model): if imap_server: imap_server.close() imap_server.logout() - if last_date: - vals = {'last_internal_date': last_date} - server.write(vals) + if last_date: + _logger.info( + "Fetched %d email(s) on %s server %s; \ + %d succeeded, %d failed.", count, + server.type, server.name, + (count - failed), failed) + vals = {'last_internal_date': last_date} + server.write(vals) return super(FetchmailServer, self).fetch_mail( cr, uid, ids, context=context) From 76ed9d965c4416760a97468b64eb78155aa6fb97 Mon Sep 17 00:00:00 2001 From: archetipo Date: Tue, 10 Mar 2015 09:45:00 +0100 Subject: [PATCH 4/7] [FIX] website link --- fetchmail_bydate/__openerp__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetchmail_bydate/__openerp__.py b/fetchmail_bydate/__openerp__.py index 5b40292c7..9d47fd250 100644 --- a/fetchmail_bydate/__openerp__.py +++ b/fetchmail_bydate/__openerp__.py @@ -30,7 +30,7 @@ This module allows to fetch email by last message internal date and unseen messages. """, 'author': "Innoviu srl, Agile Business Group, Odoo Community Association (OCA)", - 'website': 'http://www.innoviu.it http://www.agilebg.com', + 'website': 'http://www.innoviu.it', 'license': 'AGPL-3', 'depends': ['fetchmail', 'mail'], "data": [ From 7bb7c495db0aaa36cecbebcc26243335954cd546 Mon Sep 17 00:00:00 2001 From: Lorenzo Battistini Date: Wed, 11 Mar 2015 15:08:20 +0100 Subject: [PATCH 5/7] [REF] last_download_date --- fetchmail_bydate/model/fetchmail.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/fetchmail_bydate/model/fetchmail.py b/fetchmail_bydate/model/fetchmail.py index acc417711..44fdc3a3e 100644 --- a/fetchmail_bydate/model/fetchmail.py +++ b/fetchmail_bydate/model/fetchmail.py @@ -36,7 +36,7 @@ class FetchmailServer(orm.Model): _inherit = "fetchmail.server" _columns = { - 'last_internal_date': fields.datetime('Last Internal Date'), + 'last_download_date': fields.datetime('Last Download Date'), } def _fetch_from_data_imap(self, cr, uid, @@ -47,13 +47,11 @@ class FetchmailServer(orm.Model): messages = [] date_uids = {} last_date = False - last_internal_date = datetime.strptime( - server.last_internal_date, "%Y-%m-%d %H:%M:%S") - #~ timestamp1 = time.mktime(last_internal_date.timetuple()) - #~ intDate = imaplib.Time2Internaldate(timestamp1) + last_download_date = datetime.strptime( + server.last_download_date, "%Y-%m-%d %H:%M:%S") search_status, uids = imap_server.search( None, - 'SINCE', '%s' % last_internal_date.strftime('%d-%b-%Y') + 'SINCE', '%s' % last_download_date.strftime('%d-%b-%Y') ) new_uids = uids[0].split() for new_uid in new_uids: @@ -65,7 +63,7 @@ class FetchmailServer(orm.Model): internaldate_msg = datetime.fromtimestamp( time.mktime(internaldate) ) - if internaldate_msg > last_internal_date: + if internaldate_msg > last_download_date: messages.append(new_uid) date_uids[new_uid] = internaldate_msg for num in messages: @@ -124,7 +122,7 @@ class FetchmailServer(orm.Model): count, failed = 0, 0 last_date = False imap_server = False - if server.type == 'imap' and server.last_internal_date: + if server.type == 'imap' and server.last_download_date: try: imap_server = server.connect() imap_server.select() @@ -149,7 +147,7 @@ class FetchmailServer(orm.Model): %d succeeded, %d failed.", count, server.type, server.name, (count - failed), failed) - vals = {'last_internal_date': last_date} + vals = {'last_download_date': last_date} server.write(vals) return super(FetchmailServer, self).fetch_mail( cr, uid, ids, context=context) From 68a6b8217c92319346d1509d892aeff1707c7ed5 Mon Sep 17 00:00:00 2001 From: Lorenzo Battistini Date: Wed, 11 Mar 2015 16:21:38 +0100 Subject: [PATCH 6/7] [REF] _fetch_from_date_imap --- fetchmail_bydate/model/fetchmail.py | 50 +++++++++++++---------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/fetchmail_bydate/model/fetchmail.py b/fetchmail_bydate/model/fetchmail.py index 44fdc3a3e..27b1ac9b1 100644 --- a/fetchmail_bydate/model/fetchmail.py +++ b/fetchmail_bydate/model/fetchmail.py @@ -36,10 +36,10 @@ class FetchmailServer(orm.Model): _inherit = "fetchmail.server" _columns = { - 'last_download_date': fields.datetime('Last Download Date'), + 'last_internal_date': fields.datetime('Last Download Date'), } - def _fetch_from_data_imap(self, cr, uid, + def _fetch_from_date_imap(self, cr, uid, server, imap_server, mail_thread, action_pool, count, failed, @@ -47,23 +47,23 @@ class FetchmailServer(orm.Model): messages = [] date_uids = {} last_date = False - last_download_date = datetime.strptime( - server.last_download_date, "%Y-%m-%d %H:%M:%S") + last_internal_date = datetime.strptime( + server.last_internal_date, "%Y-%m-%d %H:%M:%S") search_status, uids = imap_server.search( None, - 'SINCE', '%s' % last_download_date.strftime('%d-%b-%Y') + 'SINCE', '%s' % last_internal_date.strftime('%d-%b-%Y') ) new_uids = uids[0].split() for new_uid in new_uids: - fetch_status, data = imap_server.fetch( + fetch_status, date = imap_server.fetch( int(new_uid), 'INTERNALDATE' ) - internaldate = imaplib.Internaldate2tuple(data[0]) + internaldate = imaplib.Internaldate2tuple(date[0]) internaldate_msg = datetime.fromtimestamp( time.mktime(internaldate) ) - if internaldate_msg > last_download_date: + if internaldate_msg > last_internal_date: messages.append(new_uid) date_uids[new_uid] = internaldate_msg for num in messages: @@ -71,7 +71,6 @@ class FetchmailServer(orm.Model): # recent message, even if it has already been synced res_id = None result, data = imap_server.fetch(num, '(RFC822)') - imap_server.store(num, '-FLAGS', '\\Seen') if data and data[0]: try: res_id = mail_thread.message_process( @@ -98,35 +97,32 @@ class FetchmailServer(orm.Model): "thread_model", server.object_id.model) }, context=context) - imap_server.store(num, '+FLAGS', '\\Seen') - cr.commit() - count += 1 - last_date = not failed and date_uids[num] or False + imap_server.store(num, '+FLAGS', '\\Seen') + cr.commit() + count += 1 + last_date = not failed and date_uids[num] or False return count, failed, last_date def fetch_mail(self, cr, uid, ids, context=None): - """WARNING: meant for cron usage only - - will commit() after each email! - """ if context is None: context = {} context['fetchmail_cron_running'] = True mail_thread = self.pool.get('mail.thread') action_pool = self.pool.get('ir.actions.server') for server in self.browse(cr, uid, ids, context=context): - _logger.info( - 'start checking for new emails by date on %s server %s', - server.type, server.name) - context.update({'fetchmail_server_id': server.id, - 'server_type': server.type}) - count, failed = 0, 0 - last_date = False - imap_server = False - if server.type == 'imap' and server.last_download_date: + if server.type == 'imap' and server.last_internal_date: + _logger.info( + 'start checking for new emails by date on %s server %s', + server.type, server.name) + context.update({'fetchmail_server_id': server.id, + 'server_type': server.type}) + count, failed = 0, 0 + last_date = False + imap_server = False try: imap_server = server.connect() imap_server.select() - count, failed, last_date = self._fetch_from_data_imap( + count, failed, last_date = self._fetch_from_date_imap( cr, uid, server, imap_server, mail_thread, action_pool, count, failed, context=context ) @@ -147,7 +143,7 @@ class FetchmailServer(orm.Model): %d succeeded, %d failed.", count, server.type, server.name, (count - failed), failed) - vals = {'last_download_date': last_date} + vals = {'last_internal_date': last_date} server.write(vals) return super(FetchmailServer, self).fetch_mail( cr, uid, ids, context=context) From 200863a416e090c592777eda9127b0b4f5c1c658 Mon Sep 17 00:00:00 2001 From: Lorenzo Battistini Date: Wed, 11 Mar 2015 16:27:14 +0100 Subject: [PATCH 7/7] [FIX] descriptions and copyright --- fetchmail_bydate/__openerp__.py | 16 +++++++++++----- fetchmail_bydate/model/__init__.py | 2 +- fetchmail_bydate/model/fetchmail.py | 7 +++++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/fetchmail_bydate/__openerp__.py b/fetchmail_bydate/__openerp__.py index 9d47fd250..e3b2cfa34 100644 --- a/fetchmail_bydate/__openerp__.py +++ b/fetchmail_bydate/__openerp__.py @@ -25,12 +25,18 @@ 'name': "Fetchmail by Date", "version": "1.0", 'category': 'Mailing', - 'summary': 'This module allows to fetch email by last message internal date and unseen messages.', + 'summary': 'Fetchmail by date and unseen messages', 'description': """ - This module allows to fetch email by last message internal date and unseen messages. - """, - 'author': "Innoviu srl, Agile Business Group, Odoo Community Association (OCA)", - 'website': 'http://www.innoviu.it', +This module allows to fetch new emails (using IMAP) using their date, +in addition to 'unseen' status. + +To enable this, you have to set a 'Last Download Date' in the fetchmail.server +After that, emails with an internal date greater than the saved one will be +downloaded. +""", + 'author': "Innoviu, Agile Business Group, " + "Odoo Community Association (OCA)", + 'website': 'http://www.innoviu.com', 'license': 'AGPL-3', 'depends': ['fetchmail', 'mail'], "data": [ diff --git a/fetchmail_bydate/model/__init__.py b/fetchmail_bydate/model/__init__.py index dd501ffb4..8cf9e9c84 100644 --- a/fetchmail_bydate/model/__init__.py +++ b/fetchmail_bydate/model/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- ############################################################################## # -# Copyright (C) 2015 Innoviu srl (). +# Copyright (C) 2015 Innoviu srl (). # Copyright (C) 2015 Agile Business Group http://www.agilebg.com # @authors # Roberto Onnis diff --git a/fetchmail_bydate/model/fetchmail.py b/fetchmail_bydate/model/fetchmail.py index 27b1ac9b1..a17c65dfb 100644 --- a/fetchmail_bydate/model/fetchmail.py +++ b/fetchmail_bydate/model/fetchmail.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- ############################################################################## # -# Copyright (C) 2015 Innoviu srl (). +# Copyright (C) 2015 Innoviu srl (). # Copyright (C) 2015 Agile Business Group http://www.agilebg.com # @authors # Roberto Onnis @@ -36,7 +36,10 @@ class FetchmailServer(orm.Model): _inherit = "fetchmail.server" _columns = { - 'last_internal_date': fields.datetime('Last Download Date'), + 'last_internal_date': fields.datetime( + 'Last Download Date', + help="Remote emails with a date greater than this will be " + "downloaded. Only available with IMAP"), } def _fetch_from_date_imap(self, cr, uid,