From 0920960d1e87e3eb47faa9b2de8c8faf8ed42e60 Mon Sep 17 00:00:00 2001 From: "Ronald Portier (Therp BV)" Date: Tue, 13 Sep 2022 22:46:34 +0200 Subject: [PATCH] [IMP] *_online_ponto: Transactions are read backwards from Ponto. The streamlined logic means we no longer have to store (or clear) the last identifier retrieved from Ponto. We will always read from the latest data, backward until we hit an execution date that is before the date we are interested in. --- .../__manifest__.py | 5 +- .../data/ir_cron.xml | 20 ++++++ .../online_bank_statement_provider_ponto.py | 63 +++++++++++++++---- .../models/ponto_buffer.py | 6 +- .../models/ponto_buffer_line.py | 1 + .../models/ponto_interface.py | 8 ++- .../online_bank_statement_provider.xml | 17 +++-- 7 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 account_bank_statement_import_online_ponto/data/ir_cron.xml rename account_bank_statement_import_online_ponto/{view => views}/online_bank_statement_provider.xml (52%) diff --git a/account_bank_statement_import_online_ponto/__manifest__.py b/account_bank_statement_import_online_ponto/__manifest__.py index 0a5b371..76d197c 100644 --- a/account_bank_statement_import_online_ponto/__manifest__.py +++ b/account_bank_statement_import_online_ponto/__manifest__.py @@ -1,4 +1,4 @@ -# Copyright 2020 Florent de Labarre +# Copyright 2020 Florent de Labarre. # Copyright 2022 Therp BV . # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). { @@ -14,7 +14,8 @@ "installable": True, "depends": ["account_bank_statement_import_online"], "data": [ + "data/ir_cron.xml", "security/ir.model.access.csv", - "view/online_bank_statement_provider.xml", + "views/online_bank_statement_provider.xml", ], } diff --git a/account_bank_statement_import_online_ponto/data/ir_cron.xml b/account_bank_statement_import_online_ponto/data/ir_cron.xml new file mode 100644 index 0000000..0e1b3c2 --- /dev/null +++ b/account_bank_statement_import_online_ponto/data/ir_cron.xml @@ -0,0 +1,20 @@ + + + + + + Remove old data from ponto buffers + 1 + days + -1 + code + 2019-01-01 00:20:00 + + + model._ponto_buffer_purge() + + + diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py index 7f25be7..1741336 100644 --- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py +++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py @@ -1,7 +1,8 @@ # Copyright 2020 Florent de Labarre # Copyright 2022 Therp BV . # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). -from datetime import datetime +from datetime import date, datetime +from dateutil.relativedelta import relativedelta import json import pytz @@ -16,10 +17,12 @@ _logger = logging.getLogger(__name__) class OnlineBankStatementProviderPonto(models.Model): _inherit = "online.bank.statement.provider" - ponto_last_identifier = fields.Char(readonly=True) - - def ponto_reset_last_identifier(self): - self.write({"ponto_last_identifier": False}) + ponto_buffer_retain_days = fields.Integer( + string="Number of days to keep Ponto Buffers", + default=61, + help="By default buffers will be kept for 61 days.\n" + "Set this to 0 to keep buffers indefinitely.", + ) @api.model def _get_available_services(self): @@ -31,17 +34,21 @@ class OnlineBankStatementProviderPonto(models.Model): def _pull(self, date_since, date_until): """Override pull to first retrieve data from Ponto.""" if self.service == "ponto": - self._ponto_retrieve_data() + self._ponto_retrieve_data(date_since) super()._pull(date_since, date_until) - def _ponto_retrieve_data(self): - """Fill buffer with data from Ponto.""" + def _ponto_retrieve_data(self, date_since): + """Fill buffer with data from Ponto. + + We will retrieve data from the latest transactions present in Ponto + backwards, until we find data that has an execution date before date_since. + """ interface_model = self.env["ponto.interface"] buffer_model = self.env["ponto.buffer"] access_data = interface_model._login(self.username, self.password) interface_model._set_access_account(access_data, self.account_number) interface_model._ponto_synchronisation(access_data) - latest_identifier = self.ponto_last_identifier + latest_identifier = False transactions = interface_model._get_transactions( access_data, latest_identifier @@ -49,6 +56,9 @@ class OnlineBankStatementProviderPonto(models.Model): while transactions: buffer_model.sudo()._store_transactions(self, transactions) latest_identifier = transactions[-1].get("id") + earliest_datetime = self._ponto_get_execution_datetime(transactions[-1]) + if earliest_datetime < date_since: + break transactions = interface_model._get_transactions( access_data, latest_identifier @@ -107,7 +117,7 @@ class OnlineBankStatementProviderPonto(models.Model): if attributes.get(x) ] ref = " ".join(ref_list) - date = self._ponto_date_from_string(attributes.get("executionDate")) + date = self._ponto_get_execution_datetime(transaction) vals_line = { "sequence": sequence, "date": date, @@ -122,10 +132,41 @@ class OnlineBankStatementProviderPonto(models.Model): vals_line["partner_name"] = attributes["counterpartName"] return vals_line - def _ponto_date_from_string(self, date_str): + def _ponto_get_execution_datetime(self, transaction): + """Get execution datetime for a transaction. + + Odoo often names variables containing date and time just xxx_date or + date_xxx. We try to avoid this misleading naming by using datetime as + much for variables and fields of type datetime. + """ + attributes = transaction.get("attributes", {}) + return self._ponto_datetime_from_string(attributes.get("executionDate")) + + def _ponto_datetime_from_string(self, date_str): """Dates in Ponto are expressed in UTC, so we need to convert them to supplied tz for proper classification. """ dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ") dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc")) return dt.replace(tzinfo=None) + + def _ponto_buffer_purge(self): + """Remove buffers from Ponto no longer needed to import statements.""" + _logger.info("Scheduled purge of old ponto buffers...") + today = date.today() + buffer_model = self.env["ponto.buffer"] + providers = self.search([ + ("active", "=", True), + ]) + for provider in providers: + if not provider.ponto_buffer_retain_days: + continue + cutoff_date = today - relativedelta(days=provider.ponto_buffer_retain_days) + old_buffers = buffer_model.search( + [ + ("provider_id", "=", provider.id), + ("effective_date", "<", cutoff_date), + ] + ) + old_buffers.unlink() + _logger.info("Scheduled purge of old ponto buffers complete.") diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer.py b/account_bank_statement_import_online_ponto/models/ponto_buffer.py index 5acffa3..2515730 100644 --- a/account_bank_statement_import_online_ponto/models/ponto_buffer.py +++ b/account_bank_statement_import_online_ponto/models/ponto_buffer.py @@ -24,7 +24,6 @@ class PontoBuffer(models.Model): comodel_name="ponto.buffer.line", inverse_name="buffer_id", readonly=True, - ondelete="cascade", ) def _store_transactions(self, provider, transactions): @@ -32,10 +31,7 @@ class PontoBuffer(models.Model): # Start by sorting all transactions per date. transactions_per_date = {} for transaction in transactions: - ponto_execution_date = transaction.get( - "attributes", {} - ).get("executionDate") - effective_date_time = provider._ponto_date_from_string(ponto_execution_date) + effective_date_time = provider._ponto_get_execution_datetime(transaction) transaction["effective_date_time"] = effective_date_time.isoformat() key = effective_date_time.isoformat()[0:10] if key not in transactions_per_date: diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py index 5452747..fcf0a95 100644 --- a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py +++ b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py @@ -13,6 +13,7 @@ class PontoBuffer(models.Model): comodel_name="ponto.buffer", required=True, readonly=True, + ondelete="cascade", ) ponto_id = fields.Char( required=True, diff --git a/account_bank_statement_import_online_ponto/models/ponto_interface.py b/account_bank_statement_import_online_ponto/models/ponto_interface.py index 2c979d8..c84626c 100644 --- a/account_bank_statement_import_online_ponto/models/ponto_interface.py +++ b/account_bank_statement_import_online_ponto/models/ponto_interface.py @@ -137,7 +137,13 @@ class PontoInterface(models.AbstractModel): time.sleep(40) def _get_transactions(self, access_data, last_identifier): - """Get transactions from ponto, using last_identifier as pointer.""" + """Get transactions from ponto, using last_identifier as pointer. + + Note that Ponto has the transactions in descending order. The first + transaction, retrieved by not passing an identifier, is the latest + present in Ponto. If you read transactions 'after' a certain identifier + (Ponto id), you will get transactions with an earlier date. + """ url = ( PONTO_ENDPOINT + "/accounts/" diff --git a/account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml b/account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml similarity index 52% rename from account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml rename to account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml index e856cf5..93e98e8 100644 --- a/account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml +++ b/account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml @@ -6,12 +6,17 @@ - - - - -