From dd61075792f3dd0c6cb6c390872a01d23da4706d Mon Sep 17 00:00:00 2001 From: Alexey Pelykh Date: Fri, 17 Apr 2020 08:21:44 +0200 Subject: [PATCH] [IMP] account_bank_statement_import_online: tz --- .../models/online_bank_statement_provider.py | 31 +++- .../online_bank_statement_provider_dummy.py | 17 +- ...st_account_bank_statement_import_online.py | 162 +++++++++++++++++- .../views/online_bank_statement_provider.xml | 1 + 4 files changed, 207 insertions(+), 4 deletions(-) diff --git a/account_bank_statement_import_online/models/online_bank_statement_provider.py b/account_bank_statement_import_online/models/online_bank_statement_provider.py index f2431a0..ee1cbcb 100644 --- a/account_bank_statement_import_online/models/online_bank_statement_provider.py +++ b/account_bank_statement_import_online/models/online_bank_statement_provider.py @@ -2,13 +2,16 @@ # Copyright 2019-2020 Dataplug (https://dataplug.io) # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). +from datetime import datetime from dateutil.relativedelta import relativedelta, MO from decimal import Decimal import logging +from pytz import timezone, utc from sys import exc_info from odoo import models, fields, api, _ from odoo.addons.base.models.res_bank import sanitize_account_number +from odoo.addons.base.models.res_partner import _tz_get _logger = logging.getLogger(__name__) @@ -43,6 +46,15 @@ class OnlineBankStatementProvider(models.Model): account_number = fields.Char( related='journal_id.bank_account_id.sanitized_acc_number' ) + tz = fields.Selection( + selection=_tz_get, + string='Timezone', + default=lambda self: self.env.context.get('tz'), + help=( + 'Timezone to convert transaction timestamps to prior being' + ' saved into a statement.' + ), + ) service = fields.Selection( selection=lambda self: self._selection_service(), required=True, @@ -156,6 +168,7 @@ class OnlineBankStatementProvider(models.Model): ) AccountBankStatementLine = self.env['account.bank.statement.line'] for provider in self: + provider_tz = timezone(provider.tz) if provider.tz else utc statement_date_since = provider._get_statement_date_since( date_since ) @@ -225,7 +238,14 @@ class OnlineBankStatementProvider(models.Model): ) filtered_lines = [] for line_values in lines_data: - date = fields.Datetime.from_string(line_values['date']) + date = line_values['date'] + if not isinstance(date, datetime): + date = fields.Datetime.from_string(date) + + if date.tzinfo is None: + date = date.replace(tzinfo=utc) + date = date.astimezone(utc).replace(tzinfo=None) + if date < statement_date_since or date < date_since: if 'balance_start' in statement_values: statement_values['balance_start'] = ( @@ -246,6 +266,11 @@ class OnlineBankStatementProvider(models.Model): ) ) continue + + date = date.replace(tzinfo=utc) + date = date.astimezone(provider_tz).replace(tzinfo=None) + line_values['date'] = date + unique_import_id = line_values.get('unique_import_id') if unique_import_id: unique_import_id = provider._generate_unique_import_id( @@ -258,6 +283,7 @@ class OnlineBankStatementProvider(models.Model): [('unique_import_id', '=', unique_import_id)], limit=1): continue + bank_account_number = line_values.get('account_number') if bank_account_number: line_values.update({ @@ -267,6 +293,7 @@ class OnlineBankStatementProvider(models.Model): ) ), }) + filtered_lines.append(line_values) statement_values.update({ 'line_ids': [[0, False, line] for line in filtered_lines], @@ -344,6 +371,8 @@ class OnlineBankStatementProvider(models.Model): # NOTE: Statement date is treated by Odoo as start of period. Details # - addons/account/models/account_journal_dashboard.py # - def get_line_graph_datas() + tz = timezone(self.tz) if self.tz else utc + date_since = date_since.replace(tzinfo=utc).astimezone(tz) return date_since.date() @api.multi diff --git a/account_bank_statement_import_online/tests/online_bank_statement_provider_dummy.py b/account_bank_statement_import_online/tests/online_bank_statement_provider_dummy.py index fb57a75..93b812e 100644 --- a/account_bank_statement_import_online/tests/online_bank_statement_provider_dummy.py +++ b/account_bank_statement_import_online/tests/online_bank_statement_provider_dummy.py @@ -4,9 +4,10 @@ from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta +from pytz import timezone from random import randrange -from odoo import models, api +from odoo import api, fields, models class OnlineBankStatementProviderDummy(models.Model): @@ -43,6 +44,13 @@ class OnlineBankStatementProviderDummy(models.Model): randrange(-10000, 10000, 1) * 0.1 ) balance = balance_start + + tz = self.env.context.get('tz') + if tz: + tz = timezone(tz) + + timestamp_mode = self.env.context.get('timestamp_mode') + lines = [] date = data_since while date < data_until: @@ -50,10 +58,15 @@ class OnlineBankStatementProviderDummy(models.Model): 'amount', randrange(-100, 100, 1) * 0.1 ) + transaction_date = date.replace(tzinfo=tz) + if timestamp_mode == 'date': + transaction_date = transaction_date.date() + elif timestamp_mode == 'str': + transaction_date = fields.Datetime.to_string(transaction_date) lines.append({ 'name': 'payment', 'amount': amount, - 'date': date, + 'date': transaction_date, 'unique_import_id': str(int( (date - datetime(1970, 1, 1)) / timedelta(seconds=1) )), diff --git a/account_bank_statement_import_online/tests/test_account_bank_statement_import_online.py b/account_bank_statement_import_online/tests/test_account_bank_statement_import_online.py index e1780a0..e868519 100644 --- a/account_bank_statement_import_online/tests/test_account_bank_statement_import_online.py +++ b/account_bank_statement_import_online/tests/test_account_bank_statement_import_online.py @@ -2,7 +2,7 @@ # Copyright 2019-2020 Dataplug (https://dataplug.io) # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). -from datetime import datetime +from datetime import date, datetime from dateutil.relativedelta import relativedelta from psycopg2 import IntegrityError from urllib.error import HTTPError @@ -384,6 +384,7 @@ class TestAccountBankAccountStatementImportOnline(common.TransactionCase): provider.with_context( step={'hours': 2}, balance_start=0, + amount=100.0, balance=False, )._pull( self.now - relativedelta(days=1), @@ -483,3 +484,162 @@ class TestAccountBankAccountStatementImportOnline(common.TransactionCase): self.assertEqual(statements[0].balance_end_real, 31.0) self.assertEqual(statements[1].balance_start, 31.0) self.assertEqual(statements[1].balance_end_real, 59.0) + + def test_tz_utc(self): + journal = self.AccountJournal.create({ + 'name': 'Bank', + 'type': 'bank', + 'code': 'BANK', + 'bank_statements_source': 'online', + 'online_bank_statement_provider': 'dummy', + }) + + provider = journal.online_bank_statement_provider_id + provider.active = True + provider.tz = 'UTC' + provider.with_context( + step={'hours': 1}, + tz='UTC', + )._pull( + datetime(2020, 4, 17, 22, 0), + datetime(2020, 4, 18, 2, 0), + ) + + statement = self.AccountBankStatement.search( + [('journal_id', '=', journal.id)], + ) + self.assertEqual(len(statement), 2) + + lines = statement.mapped('line_ids').sorted() + self.assertEqual(len(lines), 4) + self.assertEqual(lines[0].date, date(2020, 4, 17)) + self.assertEqual(lines[1].date, date(2020, 4, 17)) + self.assertEqual(lines[2].date, date(2020, 4, 18)) + self.assertEqual(lines[3].date, date(2020, 4, 18)) + + def test_tz_non_utc(self): + journal = self.AccountJournal.create({ + 'name': 'Bank', + 'type': 'bank', + 'code': 'BANK', + 'bank_statements_source': 'online', + 'online_bank_statement_provider': 'dummy', + }) + + provider = journal.online_bank_statement_provider_id + provider.active = True + provider.tz = 'Etc/GMT-2' + provider.with_context( + step={'hours': 1}, + tz='UTC', + )._pull( + datetime(2020, 4, 17, 22, 0), + datetime(2020, 4, 18, 2, 0), + ) + + statement = self.AccountBankStatement.search( + [('journal_id', '=', journal.id)], + ) + self.assertEqual(len(statement), 2) + + lines = statement.mapped('line_ids').sorted() + self.assertEqual(len(lines), 4) + self.assertEqual(lines[0].date, date(2020, 4, 18)) + self.assertEqual(lines[1].date, date(2020, 4, 18)) + self.assertEqual(lines[2].date, date(2020, 4, 18)) + self.assertEqual(lines[3].date, date(2020, 4, 18)) + + def test_other_tz_to_utc(self): + journal = self.AccountJournal.create({ + 'name': 'Bank', + 'type': 'bank', + 'code': 'BANK', + 'bank_statements_source': 'online', + 'online_bank_statement_provider': 'dummy', + }) + + provider = journal.online_bank_statement_provider_id + provider.active = True + provider.with_context( + step={'hours': 1}, + tz='Etc/GMT-2', + data_since=datetime(2020, 4, 18, 0, 0), + data_until=datetime(2020, 4, 18, 4, 0), + )._pull( + datetime(2020, 4, 17, 22, 0), + datetime(2020, 4, 18, 2, 0), + ) + + statement = self.AccountBankStatement.search( + [('journal_id', '=', journal.id)], + ) + self.assertEqual(len(statement), 2) + + lines = statement.mapped('line_ids').sorted() + self.assertEqual(len(lines), 4) + self.assertEqual(lines[0].date, date(2020, 4, 17)) + self.assertEqual(lines[1].date, date(2020, 4, 17)) + self.assertEqual(lines[2].date, date(2020, 4, 18)) + self.assertEqual(lines[3].date, date(2020, 4, 18)) + + def test_timestamp_date_only(self): + journal = self.AccountJournal.create({ + 'name': 'Bank', + 'type': 'bank', + 'code': 'BANK', + 'bank_statements_source': 'online', + 'online_bank_statement_provider': 'dummy', + }) + + provider = journal.online_bank_statement_provider_id + provider.active = True + provider.with_context( + step={'hours': 1}, + timestamp_mode='date', + )._pull( + datetime(2020, 4, 18, 0, 0), + datetime(2020, 4, 18, 4, 0), + ) + + statement = self.AccountBankStatement.search( + [('journal_id', '=', journal.id)], + ) + self.assertEqual(len(statement), 1) + + lines = statement.line_ids + self.assertEqual(len(lines), 4) + self.assertEqual(lines[0].date, date(2020, 4, 18)) + self.assertEqual(lines[1].date, date(2020, 4, 18)) + self.assertEqual(lines[2].date, date(2020, 4, 18)) + self.assertEqual(lines[3].date, date(2020, 4, 18)) + + def test_timestamp_date_only(self): + journal = self.AccountJournal.create({ + 'name': 'Bank', + 'type': 'bank', + 'code': 'BANK', + 'bank_statements_source': 'online', + 'online_bank_statement_provider': 'dummy', + }) + + provider = journal.online_bank_statement_provider_id + provider.active = True + provider.with_context( + step={'hours': 1}, + timestamp_mode='str', + )._pull( + datetime(2020, 4, 18, 0, 0), + datetime(2020, 4, 18, 4, 0), + ) + + statement = self.AccountBankStatement.search( + [('journal_id', '=', journal.id)], + ) + self.assertEqual(len(statement), 1) + + lines = statement.line_ids + self.assertEqual(len(lines), 4) + self.assertEqual(lines[0].date, date(2020, 4, 18)) + self.assertEqual(lines[1].date, date(2020, 4, 18)) + self.assertEqual(lines[2].date, date(2020, 4, 18)) + self.assertEqual(lines[3].date, date(2020, 4, 18)) diff --git a/account_bank_statement_import_online/views/online_bank_statement_provider.xml b/account_bank_statement_import_online/views/online_bank_statement_provider.xml index 232ba04..5fb927e 100644 --- a/account_bank_statement_import_online/views/online_bank_statement_provider.xml +++ b/account_bank_statement_import_online/views/online_bank_statement_provider.xml @@ -80,6 +80,7 @@ +