[IMP] *_online_ponto: Transactions are read backwards from Ponto.

The streamlined logic means we no longer have to store (or clear) the
last identifier retrieved from Ponto. We will always read from the latest data,
backward until we hit an execution date that is before the date we are interested in.

account_bank_statement_import_online_ponto/__manifest__.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Florent de Labarre
+# Copyright 2020 Florent de Labarre.
 # Copyright 2022 Therp BV <https://therp.nl>.
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 {
@@ -14,7 +14,8 @@
     "installable": True,
     "depends": ["account_bank_statement_import_online"],
     "data": [
+        "data/ir_cron.xml",
         "security/ir.model.access.csv",
-        "view/online_bank_statement_provider.xml",
+        "views/online_bank_statement_provider.xml",
     ],
 }

account_bank_statement_import_online_ponto/data/ir_cron.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" ?>
+<!--
+    Copyright 2022 Therp BV (https://therp.nl)
+    License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+-->
+<odoo noupdate="1">
+
+    <record model="ir.cron" id="ir_cron_purge_ponto_buffer">
+        <field name="name">Remove old data from ponto buffers</field>
+        <field name="interval_number">1</field>
+        <field name="interval_type">days</field>
+        <field name="numbercall">-1</field>
+        <field name="state">code</field>
+        <field name="nextcall">2019-01-01 00:20:00</field>
+        <field name="doall" eval="False"/>
+        <field name="model_id" ref="account_bank_statement_import_online_ponto.model_online_bank_statement_provider"/>
+        <field name="code">model._ponto_buffer_purge()</field>
+    </record>
+
+</odoo>

account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py

@@ -1,7 +1,8 @@
 # Copyright 2020 Florent de Labarre
 # Copyright 2022 Therp BV <https://therp.nl>.
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-from datetime import datetime
+from datetime import date, datetime
+from dateutil.relativedelta import relativedelta
 import json
 import pytz
 
@@ -16,10 +17,12 @@ _logger = logging.getLogger(__name__)
 class OnlineBankStatementProviderPonto(models.Model):
     _inherit = "online.bank.statement.provider"
 
-    ponto_last_identifier = fields.Char(readonly=True)
-
-    def ponto_reset_last_identifier(self):
-        self.write({"ponto_last_identifier": False})
+    ponto_buffer_retain_days = fields.Integer(
+        string="Number of days to keep Ponto Buffers",
+        default=61,
+        help="By default buffers will be kept for 61 days.\n"
+        "Set this to 0 to keep buffers indefinitely.",
+    )
 
     @api.model
     def _get_available_services(self):
@@ -31,17 +34,21 @@ class OnlineBankStatementProviderPonto(models.Model):
     def _pull(self, date_since, date_until):
         """Override pull to first retrieve data from Ponto."""
         if self.service == "ponto":
-            self._ponto_retrieve_data()
+            self._ponto_retrieve_data(date_since)
         super()._pull(date_since, date_until)
 
-    def _ponto_retrieve_data(self):
-        """Fill buffer with data from Ponto."""
+    def _ponto_retrieve_data(self, date_since):
+        """Fill buffer with data from Ponto.
+
+        We will retrieve data from the latest transactions present in Ponto
+        backwards, until we find data that has an execution date before date_since.
+        """
         interface_model = self.env["ponto.interface"]
         buffer_model = self.env["ponto.buffer"]
         access_data = interface_model._login(self.username, self.password)
         interface_model._set_access_account(access_data, self.account_number)
         interface_model._ponto_synchronisation(access_data)
-        latest_identifier = self.ponto_last_identifier
+        latest_identifier = False
         transactions = interface_model._get_transactions(
             access_data,
             latest_identifier
@@ -49,6 +56,9 @@ class OnlineBankStatementProviderPonto(models.Model):
         while transactions:
             buffer_model.sudo()._store_transactions(self, transactions)
             latest_identifier = transactions[-1].get("id")
+            earliest_datetime = self._ponto_get_execution_datetime(transactions[-1])
+            if earliest_datetime < date_since:
+                break
             transactions = interface_model._get_transactions(
                 access_data,
                 latest_identifier
@@ -107,7 +117,7 @@ class OnlineBankStatementProviderPonto(models.Model):
             if attributes.get(x)
         ]
         ref = " ".join(ref_list)
-        date = self._ponto_date_from_string(attributes.get("executionDate"))
+        date = self._ponto_get_execution_datetime(transaction)
         vals_line = {
             "sequence": sequence,
             "date": date,
@@ -122,10 +132,41 @@ class OnlineBankStatementProviderPonto(models.Model):
             vals_line["partner_name"] = attributes["counterpartName"]
         return vals_line
 
-    def _ponto_date_from_string(self, date_str):
+    def _ponto_get_execution_datetime(self, transaction):
+        """Get execution datetime for a transaction.
+
+        Odoo often names variables containing date and time just xxx_date or
+        date_xxx. We try to avoid this misleading naming by using datetime as
+        much for variables and fields of type datetime.
+        """
+        attributes = transaction.get("attributes", {})
+        return self._ponto_datetime_from_string(attributes.get("executionDate"))
+
+    def _ponto_datetime_from_string(self, date_str):
         """Dates in Ponto are expressed in UTC, so we need to convert them
         to supplied tz for proper classification.
         """
         dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ")
         dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc"))
         return dt.replace(tzinfo=None)
+
+    def _ponto_buffer_purge(self):
+        """Remove buffers from Ponto no longer needed to import statements."""
+        _logger.info("Scheduled purge of old ponto buffers...")
+        today = date.today()
+        buffer_model = self.env["ponto.buffer"]
+        providers = self.search([
+            ("active", "=", True),
+        ])
+        for provider in providers:
+            if not provider.ponto_buffer_retain_days:
+                continue
+            cutoff_date = today - relativedelta(days=provider.ponto_buffer_retain_days)
+            old_buffers = buffer_model.search(
+                [
+                    ("provider_id", "=", provider.id),
+                    ("effective_date", "<", cutoff_date),
+                ]
+            )
+            old_buffers.unlink()
+        _logger.info("Scheduled purge of old ponto buffers complete.")

account_bank_statement_import_online_ponto/models/ponto_buffer.py

@@ -24,7 +24,6 @@ class PontoBuffer(models.Model):
         comodel_name="ponto.buffer.line",
         inverse_name="buffer_id",
         readonly=True,
-        ondelete="cascade",
     )
 
     def _store_transactions(self, provider, transactions):
@@ -32,10 +31,7 @@ class PontoBuffer(models.Model):
         # Start by sorting all transactions per date.
         transactions_per_date = {}
         for transaction in transactions:
-            ponto_execution_date = transaction.get(
-                "attributes", {}
-            ).get("executionDate")
-            effective_date_time = provider._ponto_date_from_string(ponto_execution_date)
+            effective_date_time = provider._ponto_get_execution_datetime(transaction)
             transaction["effective_date_time"] = effective_date_time.isoformat()
             key = effective_date_time.isoformat()[0:10]
             if key not in transactions_per_date:

account_bank_statement_import_online_ponto/models/ponto_buffer_line.py

@@ -13,6 +13,7 @@ class PontoBuffer(models.Model):
         comodel_name="ponto.buffer",
         required=True,
         readonly=True,
+        ondelete="cascade",
     )
     ponto_id = fields.Char(
         required=True,

account_bank_statement_import_online_ponto/models/ponto_interface.py

@@ -137,7 +137,13 @@ class PontoInterface(models.AbstractModel):
             time.sleep(40)
 
     def _get_transactions(self, access_data, last_identifier):
-        """Get transactions from ponto, using last_identifier as pointer."""
+        """Get transactions from ponto, using last_identifier as pointer.
+
+        Note that Ponto has the transactions in descending order. The first
+        transaction, retrieved by not passing an identifier, is the latest
+        present in Ponto. If you read transactions 'after' a certain identifier
+        (Ponto id), you will get transactions with an earlier date.
+        """
         url = (
             PONTO_ENDPOINT
             + "/accounts/"

account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml → account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml

@@ -6,12 +6,17 @@
         <field name="inherit_id" ref="account_bank_statement_import_online.online_bank_statement_provider_form"/>
         <field name="arch" type="xml">
             <xpath expr="//page[@name='configuration']" position="inside">
-                <group name="qonto" attrs="{'invisible':[('service','!=','ponto')]}">
-                    <field name="username" string="Login"/>
-                    <field name="password" string="Secret Key"/>
-                    <field name="ponto_last_identifier"/>
-                    <button name="ponto_reset_last_identifier" string="Reset Last identifier." type="object"
-                            attrs="{'invisible':[('ponto_last_identifier','=',False)]}"/>
+                <group
+                    name="ponto_configuration"
+                    attrs="{'invisible':[('service','!=','ponto')]}"
+                    colspan="6"
+                    col="3"
+                >
+                    <group colspan="2" col="2" name="ponto_login">
+                        <field name="username" string="Login"/>
+                        <field name="password" string="Secret Key"/>
+                        <field name="ponto_buffer_retain_days"/>
+                    </group>
                 </group>
             </xpath>
         </field>