diff --git a/account_bank_statement_import_online_ponto/__manifest__.py b/account_bank_statement_import_online_ponto/__manifest__.py index 0a5b371..76d197c 100644 --- a/account_bank_statement_import_online_ponto/__manifest__.py +++ b/account_bank_statement_import_online_ponto/__manifest__.py @@ -1,4 +1,4 @@ -# Copyright 2020 Florent de Labarre +# Copyright 2020 Florent de Labarre. # Copyright 2022 Therp BV . # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). { @@ -14,7 +14,8 @@ "installable": True, "depends": ["account_bank_statement_import_online"], "data": [ + "data/ir_cron.xml", "security/ir.model.access.csv", - "view/online_bank_statement_provider.xml", + "views/online_bank_statement_provider.xml", ], } diff --git a/account_bank_statement_import_online_ponto/data/ir_cron.xml b/account_bank_statement_import_online_ponto/data/ir_cron.xml new file mode 100644 index 0000000..0e1b3c2 --- /dev/null +++ b/account_bank_statement_import_online_ponto/data/ir_cron.xml @@ -0,0 +1,20 @@ + + + + + + Remove old data from ponto buffers + 1 + days + -1 + code + 2019-01-01 00:20:00 + + + model._ponto_buffer_purge() + + + diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py index 7f25be7..1741336 100644 --- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py +++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py @@ -1,7 +1,8 @@ # Copyright 2020 Florent de Labarre # Copyright 2022 Therp BV . # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). -from datetime import datetime +from datetime import date, datetime +from dateutil.relativedelta import relativedelta import json import pytz @@ -16,10 +17,12 @@ _logger = logging.getLogger(__name__) class OnlineBankStatementProviderPonto(models.Model): _inherit = "online.bank.statement.provider" - ponto_last_identifier = fields.Char(readonly=True) - - def ponto_reset_last_identifier(self): - self.write({"ponto_last_identifier": False}) + ponto_buffer_retain_days = fields.Integer( + string="Number of days to keep Ponto Buffers", + default=61, + help="By default buffers will be kept for 61 days.\n" + "Set this to 0 to keep buffers indefinitely.", + ) @api.model def _get_available_services(self): @@ -31,17 +34,21 @@ class OnlineBankStatementProviderPonto(models.Model): def _pull(self, date_since, date_until): """Override pull to first retrieve data from Ponto.""" if self.service == "ponto": - self._ponto_retrieve_data() + self._ponto_retrieve_data(date_since) super()._pull(date_since, date_until) - def _ponto_retrieve_data(self): - """Fill buffer with data from Ponto.""" + def _ponto_retrieve_data(self, date_since): + """Fill buffer with data from Ponto. + + We will retrieve data from the latest transactions present in Ponto + backwards, until we find data that has an execution date before date_since. + """ interface_model = self.env["ponto.interface"] buffer_model = self.env["ponto.buffer"] access_data = interface_model._login(self.username, self.password) interface_model._set_access_account(access_data, self.account_number) interface_model._ponto_synchronisation(access_data) - latest_identifier = self.ponto_last_identifier + latest_identifier = False transactions = interface_model._get_transactions( access_data, latest_identifier @@ -49,6 +56,9 @@ class OnlineBankStatementProviderPonto(models.Model): while transactions: buffer_model.sudo()._store_transactions(self, transactions) latest_identifier = transactions[-1].get("id") + earliest_datetime = self._ponto_get_execution_datetime(transactions[-1]) + if earliest_datetime < date_since: + break transactions = interface_model._get_transactions( access_data, latest_identifier @@ -107,7 +117,7 @@ class OnlineBankStatementProviderPonto(models.Model): if attributes.get(x) ] ref = " ".join(ref_list) - date = self._ponto_date_from_string(attributes.get("executionDate")) + date = self._ponto_get_execution_datetime(transaction) vals_line = { "sequence": sequence, "date": date, @@ -122,10 +132,41 @@ class OnlineBankStatementProviderPonto(models.Model): vals_line["partner_name"] = attributes["counterpartName"] return vals_line - def _ponto_date_from_string(self, date_str): + def _ponto_get_execution_datetime(self, transaction): + """Get execution datetime for a transaction. + + Odoo often names variables containing date and time just xxx_date or + date_xxx. We try to avoid this misleading naming by using datetime as + much for variables and fields of type datetime. + """ + attributes = transaction.get("attributes", {}) + return self._ponto_datetime_from_string(attributes.get("executionDate")) + + def _ponto_datetime_from_string(self, date_str): """Dates in Ponto are expressed in UTC, so we need to convert them to supplied tz for proper classification. """ dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ") dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc")) return dt.replace(tzinfo=None) + + def _ponto_buffer_purge(self): + """Remove buffers from Ponto no longer needed to import statements.""" + _logger.info("Scheduled purge of old ponto buffers...") + today = date.today() + buffer_model = self.env["ponto.buffer"] + providers = self.search([ + ("active", "=", True), + ]) + for provider in providers: + if not provider.ponto_buffer_retain_days: + continue + cutoff_date = today - relativedelta(days=provider.ponto_buffer_retain_days) + old_buffers = buffer_model.search( + [ + ("provider_id", "=", provider.id), + ("effective_date", "<", cutoff_date), + ] + ) + old_buffers.unlink() + _logger.info("Scheduled purge of old ponto buffers complete.") diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer.py b/account_bank_statement_import_online_ponto/models/ponto_buffer.py index 5acffa3..2515730 100644 --- a/account_bank_statement_import_online_ponto/models/ponto_buffer.py +++ b/account_bank_statement_import_online_ponto/models/ponto_buffer.py @@ -24,7 +24,6 @@ class PontoBuffer(models.Model): comodel_name="ponto.buffer.line", inverse_name="buffer_id", readonly=True, - ondelete="cascade", ) def _store_transactions(self, provider, transactions): @@ -32,10 +31,7 @@ class PontoBuffer(models.Model): # Start by sorting all transactions per date. transactions_per_date = {} for transaction in transactions: - ponto_execution_date = transaction.get( - "attributes", {} - ).get("executionDate") - effective_date_time = provider._ponto_date_from_string(ponto_execution_date) + effective_date_time = provider._ponto_get_execution_datetime(transaction) transaction["effective_date_time"] = effective_date_time.isoformat() key = effective_date_time.isoformat()[0:10] if key not in transactions_per_date: diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py index 5452747..fcf0a95 100644 --- a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py +++ b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py @@ -13,6 +13,7 @@ class PontoBuffer(models.Model): comodel_name="ponto.buffer", required=True, readonly=True, + ondelete="cascade", ) ponto_id = fields.Char( required=True, diff --git a/account_bank_statement_import_online_ponto/models/ponto_interface.py b/account_bank_statement_import_online_ponto/models/ponto_interface.py index 2c979d8..c84626c 100644 --- a/account_bank_statement_import_online_ponto/models/ponto_interface.py +++ b/account_bank_statement_import_online_ponto/models/ponto_interface.py @@ -137,7 +137,13 @@ class PontoInterface(models.AbstractModel): time.sleep(40) def _get_transactions(self, access_data, last_identifier): - """Get transactions from ponto, using last_identifier as pointer.""" + """Get transactions from ponto, using last_identifier as pointer. + + Note that Ponto has the transactions in descending order. The first + transaction, retrieved by not passing an identifier, is the latest + present in Ponto. If you read transactions 'after' a certain identifier + (Ponto id), you will get transactions with an earlier date. + """ url = ( PONTO_ENDPOINT + "/accounts/" diff --git a/account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml b/account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml similarity index 52% rename from account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml rename to account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml index e856cf5..93e98e8 100644 --- a/account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml +++ b/account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml @@ -6,12 +6,17 @@ - - - - -