From 26ed33bc7f071bc06a95b61ff1fd5fb242776d93 Mon Sep 17 00:00:00 2001 From: "Ronald Portier (Therp BV)" Date: Wed, 31 Aug 2022 16:40:36 +0200 Subject: [PATCH] [IMP] *_online_ponto: more efficient ponto account retrieval Also do not name variables account_id or account_ids when they do not refer to ids in the Odoo sense, nor to either bank or ledger accounts. --- .../__manifest__.py | 8 +- .../models/__init__.py | 5 +- .../online_bank_statement_provider_ponto.py | 261 ++++-------------- .../models/ponto_buffer.py | 79 ++++++ .../models/ponto_buffer_line.py | 28 ++ .../models/ponto_interface.py | 190 +++++++++++++ 6 files changed, 360 insertions(+), 211 deletions(-) create mode 100644 account_bank_statement_import_online_ponto/models/ponto_buffer.py create mode 100644 account_bank_statement_import_online_ponto/models/ponto_buffer_line.py create mode 100644 account_bank_statement_import_online_ponto/models/ponto_interface.py diff --git a/account_bank_statement_import_online_ponto/__manifest__.py b/account_bank_statement_import_online_ponto/__manifest__.py index 1b0b849..919f0e0 100644 --- a/account_bank_statement_import_online_ponto/__manifest__.py +++ b/account_bank_statement_import_online_ponto/__manifest__.py @@ -1,11 +1,15 @@ # Copyright 2020 Florent de Labarre +# Copyright 2022 Therp BV . # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). { "name": "Online Bank Statements: MyPonto.com", - "version": "12.0.1.1.1", + "version": "12.0.1.2.0", "category": "Account", "website": "https://github.com/OCA/bank-statement-import", - "author": "Florent de Labarre, Odoo Community Association (OCA)", + "author": + "Florent de Labarre" + ", Therp BV" + ", Odoo Community Association (OCA)", "license": "AGPL-3", "installable": True, "depends": ["account_bank_statement_import_online"], diff --git a/account_bank_statement_import_online_ponto/models/__init__.py b/account_bank_statement_import_online_ponto/models/__init__.py index cc57537..4044c75 100644 --- a/account_bank_statement_import_online_ponto/models/__init__.py +++ b/account_bank_statement_import_online_ponto/models/__init__.py @@ -1,4 +1,5 @@ -# Copyright 2020 Florent de Labarre # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). - +from . import ponto_buffer +from . import ponto_buffer_line +from . import ponto_interface from . import online_bank_statement_provider_ponto diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py index b264ad7..30c6551 100644 --- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py +++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py @@ -1,31 +1,21 @@ # Copyright 2020 Florent de Labarre +# Copyright 2022 Therp BV . # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). - -import base64 +from datetime import datetime import json +import pytz + import logging import re -import time -from datetime import datetime - -import pytz -import requests -from dateutil.relativedelta import relativedelta from odoo import api, fields, models, _ -from odoo.exceptions import UserError -from odoo.addons.base.models.res_bank import sanitize_account_number _logger = logging.getLogger(__name__) -PONTO_ENDPOINT = "https://api.myponto.com" - class OnlineBankStatementProviderPonto(models.Model): _inherit = "online.bank.statement.provider" - ponto_token = fields.Char(readonly=True) - ponto_token_expiration = fields.Datetime(readonly=True) ponto_last_identifier = fields.Char(readonly=True) def ponto_reset_last_identifier(self): @@ -37,6 +27,35 @@ class OnlineBankStatementProviderPonto(models.Model): ("ponto", "MyPonto.com"), ] + @api.multi + def _pull(self, date_since, date_until): + """Override pull to first retrieve data from Ponto.""" + self.ensure_one() + if self.service == "ponto": + self._ponto_retrieve_data() + super()._pull(date_since, date_until) + + def _ponto_retrieve_data(self): + """Fill buffer with data from Ponto.""" + interface_model = self.env["ponto.interface"] + buffer_model = self.env["ponto.buffer"] + access_data = interface_model._login(self.username, self.password) + interface_model._set_access_account(access_data, self.account_number) + interface_model._ponto_synchronisation(access_data) + latest_identifier = self.ponto_last_identifier + transactions = interface_model._get_transactions( + access_data, + latest_identifier + ) + while transactions: + buffer_model.sudo()._store_transactions(self, transactions) + latest_identifier = transactions[-1].get("id") + transactions = interface_model._get_transactions( + access_data, + latest_identifier + ) + self.ponto_last_identifier = latest_identifier + def _obtain_statement_data(self, date_since, date_until): self.ensure_one() if self.service != "ponto": @@ -46,211 +65,31 @@ class OnlineBankStatementProviderPonto(models.Model): ) return self._ponto_obtain_statement_data(date_since, date_until) - ######### - # ponto # - ######### - - def _ponto_header_token(self): - self.ensure_one() - if self.username and self.password: - login = "%s:%s" % (self.username, self.password) - login = base64.b64encode(login.encode("UTF-8")).decode("UTF-8") - return { - "Content-Type": "application/x-www-form-urlencoded", - "Accept": "application/json", - "Authorization": "Basic %s" % login, - } - raise UserError(_("Please fill login and key.")) - - def _ponto_header(self): - self.ensure_one() - if ( - not self.ponto_token - or not self.ponto_token_expiration - or self.ponto_token_expiration <= fields.Datetime.now() - ): - - url = PONTO_ENDPOINT + "/oauth2/token" - response = requests.post( - url, - params={"grant_type": "client_credentials"}, - headers=self._ponto_header_token(), - ) - if response.status_code == 200: - data = json.loads(response.text) - access_token = data.get("access_token", False) - if not access_token: - raise UserError(_("Ponto : no token")) - else: - self.sudo().ponto_token = access_token - expiration_date = fields.Datetime.now() + relativedelta( - seconds=data.get("expires_in", False) - ) - self.sudo().ponto_token_expiration = expiration_date - else: - raise UserError(_("%s \n\n %s") % (response.status_code, response.text)) - return { - "Accept": "application/json", - "Authorization": "Bearer %s" % self.ponto_token, - } - - def _ponto_get_account_ids(self): - url = PONTO_ENDPOINT + "/accounts" - response = requests.get( - url, params={"limit": 100}, headers=self._ponto_header() - ) - if response.status_code == 200: - data = json.loads(response.text) - res = {} - for account in data.get("data", []): - iban = sanitize_account_number( - account.get("attributes", {}).get("reference", "") - ) - res[iban] = account.get("id") - return res - raise UserError(_("%s \n\n %s") % (response.status_code, response.text)) - - def _ponto_synchronisation(self, account_id): - url = PONTO_ENDPOINT + "/synchronizations" - data = { - "data": { - "type": "synchronization", - "attributes": { - "resourceType": "account", - "resourceId": account_id, - "subtype": "accountTransactions", - }, - } - } - response = requests.post( - url, headers=self._ponto_header(), json=data - ) - if response.status_code in (200, 201, 400): - data = json.loads(response.text) - sync_id = data.get("attributes", {}).get("resourceId", False) - else: - raise UserError( - _("Error during Create Synchronisation %s \n\n %s") - % (response.status_code, response.text) - ) - # Check synchronisation - if not sync_id: - return - url = PONTO_ENDPOINT + "/synchronizations/" + sync_id - number = 0 - while number == 100: - number += 1 - response = requests.get(url, headers=self._ponto_header()) - if response.status_code == 200: - data = json.loads(response.text) - status = data.get("status", {}) - if status in ("success", "error"): - return - time.sleep(4) - - def _ponto_get_transaction(self, account_id, date_since, date_until): - page_url = PONTO_ENDPOINT + "/accounts/" + account_id + "/transactions" - params = {"limit": 100} - page_next = True - last_identifier = self.ponto_last_identifier - if last_identifier: - params["before"] = last_identifier - page_next = False - transaction_lines = [] - latest_identifier = False - while page_url: - headers = self._ponto_header() - response = requests.get( - page_url, params=params, headers=headers - ) - _logger.debug( - _("Get request to %s, with headers %s and params %s"), - page_url, - params, - headers - ) - if response.status_code != 200: - raise UserError( - _("Error during get transaction.\n\n%s \n\n %s") - % (response.status_code, response.text) - ) - if params.get("before"): - params.pop("before") - data = json.loads(response.text) - links = data.get("links", {}) - if page_next: - page_url = links.get("next", False) - else: - page_url = links.get("prev", False) - transactions = data.get("data", []) - if not transactions: - _logger.debug( - _("No transactions where found in response %s"), - response.text, - ) - else: - _logger.debug( - _("%d transactions present in response data"), - len(transactions), - ) - current_transactions = [] - for transaction in transactions: - date = self._ponto_date_from_string( - transaction.get("attributes", {}).get("executionDate") - ) - if date_since <= date < date_until: - current_transactions.append(transaction) - if not current_transactions: - _logger.debug( - _("No lines selected from transactions") - ) - else: - _logger.debug( - _("%d lines selected from transactions"), - len(current_transactions), - ) - if not page_next or (page_next and not latest_identifier): - latest_identifier = current_transactions[0].get("id") - transaction_lines.extend(current_transactions) - if latest_identifier: - self.ponto_last_identifier = latest_identifier - return transaction_lines - - def _ponto_date_from_string(self, date_str): - """Dates in Ponto are expressed in UTC, so we need to convert them - to supplied tz for proper classification. - """ - dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ") - dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc")) - return dt.replace(tzinfo=None) - def _ponto_obtain_statement_data(self, date_since, date_until): """Translate information from Ponto to Odoo bank statement lines.""" self.ensure_one() - account_ids = self._ponto_get_account_ids() - journal = self.journal_id - iban = self.account_number - account_id = account_ids.get(iban) - if not account_id: - raise UserError( - _("Ponto : wrong configuration, unknow account %s") - % journal.bank_account_id.acc_number - ) _logger.debug( _("Ponto obtain statement data for journal %s from %s to %s"), - journal.name, + self.journal_id.name, date_since, date_until ) - self._ponto_synchronisation(account_id) - transaction_lines = self._ponto_get_transaction( - account_id, date_since, date_until + line_model = self.env["ponto.buffer.line"] + lines = line_model.sudo().search( + [ + ("buffer_id.provider_id", "=", self.id), + ("effective_date_time", ">=", date_since), + ("effective_date_time", "<=", date_until), + ] ) new_transactions = [] sequence = 0 - for transaction in transaction_lines: + for transaction in lines: sequence += 1 - vals_line = self._ponto_get_transaction_vals(transaction, sequence) + vals_line = self._ponto_get_transaction_vals( + json.loads(transaction.transaction_data), + sequence + ) new_transactions.append(vals_line) if new_transactions: return new_transactions, {} @@ -283,3 +122,11 @@ class OnlineBankStatementProviderPonto(models.Model): if attributes.get("counterpartName"): vals_line["partner_name"] = attributes["counterpartName"] return vals_line + + def _ponto_date_from_string(self, date_str): + """Dates in Ponto are expressed in UTC, so we need to convert them + to supplied tz for proper classification. + """ + dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ") + dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc")) + return dt.replace(tzinfo=None) diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer.py b/account_bank_statement_import_online_ponto/models/ponto_buffer.py new file mode 100644 index 0000000..5acffa3 --- /dev/null +++ b/account_bank_statement_import_online_ponto/models/ponto_buffer.py @@ -0,0 +1,79 @@ +# Copyright 2022 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +"""Define model to hold data retrieved from Ponto.""" +import json +import logging + +from odoo import _, fields, models + +_logger = logging.getLogger(__name__) + + +class PontoBuffer(models.Model): + """Define model to hold data retrieved from Ponto.""" + _name = "ponto.buffer" + _description = "Group transactions retrieved from Ponto." + + provider_id = fields.Many2one( + comodel_name="online.bank.statement.provider", + required=True, + readonly=True, + ) + effective_date = fields.Date(readonly=True, required=True) + buffer_line_ids = fields.One2many( + comodel_name="ponto.buffer.line", + inverse_name="buffer_id", + readonly=True, + ondelete="cascade", + ) + + def _store_transactions(self, provider, transactions): + """Store transactions retrieved from Ponto in buffer, preventing duplicates.""" + # Start by sorting all transactions per date. + transactions_per_date = {} + for transaction in transactions: + ponto_execution_date = transaction.get( + "attributes", {} + ).get("executionDate") + effective_date_time = provider._ponto_date_from_string(ponto_execution_date) + transaction["effective_date_time"] = effective_date_time.isoformat() + key = effective_date_time.isoformat()[0:10] + if key not in transactions_per_date: + transactions_per_date[key] = [] # Initialize transaction array. + transactions_per_date[key].append(transaction) + # Now store the transactions, but not when already present. + for key, date_transactions in transactions_per_date.items(): + _logger.debug( + _("For date %s we retrieved %d transactions"), + key, + len(date_transactions) + ) + ponto_buffer = self.search( + [ + ("provider_id", "=", provider.id), + ("effective_date", "=", key), + ], + limit=1 + ) or self.create( + { + "provider_id": provider.id, + "effective_date": key, + } + ) + already_present = ponto_buffer.buffer_line_ids.mapped("ponto_id") + new_lines = [ + ( + 0, + 0, + { + "buffer_id": ponto_buffer.id, + "ponto_id": t.get("id"), + "effective_date_time": t.get("effective_date_time"), + "transaction_data": json.dumps(t), + }, + ) + for t in date_transactions + if t.get("id") not in already_present + ] + if new_lines: + ponto_buffer.write({"buffer_line_ids": new_lines}) diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py new file mode 100644 index 0000000..5452747 --- /dev/null +++ b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py @@ -0,0 +1,28 @@ +# Copyright 2022 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +"""Define model to hold transactions retrieved from Ponto.""" +from odoo import fields, models + + +class PontoBuffer(models.Model): + """Define model to hold transactions retrieved from Ponto.""" + _name = "ponto.buffer.line" + _description = "Hold transactions retrieved from Ponto." + + buffer_id = fields.Many2one( + comodel_name="ponto.buffer", + required=True, + readonly=True, + ) + ponto_id = fields.Char( + required=True, + readonly=True, + ) + effective_date_time = fields.Datetime( + required=True, + readonly=True, + ) + transaction_data = fields.Char( + required=True, + readonly=True, + ) diff --git a/account_bank_statement_import_online_ponto/models/ponto_interface.py b/account_bank_statement_import_online_ponto/models/ponto_interface.py new file mode 100644 index 0000000..2c979d8 --- /dev/null +++ b/account_bank_statement_import_online_ponto/models/ponto_interface.py @@ -0,0 +1,190 @@ +# Copyright 2020 Florent de Labarre +# Copyright 2022 Therp BV . +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 +import json +import logging +import time + +import requests +from dateutil.relativedelta import relativedelta + +from odoo import fields, models, _ +from odoo.exceptions import UserError +from odoo.addons.base.models.res_bank import sanitize_account_number + +_logger = logging.getLogger(__name__) + +PONTO_ENDPOINT = "https://api.myponto.com" + + +class PontoInterface(models.AbstractModel): + _name = "ponto.interface" + _description = "Interface to all interactions with Ponto API" + + def _login(self, username, password): + """Ponto login returns an access dictionary for further requests.""" + url = PONTO_ENDPOINT + "/oauth2/token" + if not(username and password): + raise UserError(_("Please fill login and key.")) + login = "%s:%s" % (username, password) + login = base64.b64encode(login.encode("UTF-8")).decode("UTF-8") + login_headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "application/json", + "Authorization": "Basic %s" % login, + } + response = requests.post( + url, + params={"grant_type": "client_credentials"}, + headers=login_headers, + ) + data = self._get_response_data(response) + access_token = data.get("access_token", False) + if not access_token: + raise UserError(_("Ponto : no token")) + token_expiration = fields.Datetime.now() + relativedelta( + seconds=data.get("expires_in", False) + ) + return { + "username": username, + "password": password, + "access_token": access_token, + "token_expiration": token_expiration, + } + + def _get_request_headers(self, access_data): + """Get headers with authorization for further ponto requests.""" + if access_data["token_expiration"] <= fields.Datetime.now(): + updated_data = self._login(access_data["username"], access_data["password"]) + access_data.update(updated_data) + return { + "Accept": "application/json", + "Authorization": "Bearer %s" % access_data["access_token"], + } + + def _set_access_account(self, access_data, account_number): + """Set ponto account for bank account in access_data.""" + url = PONTO_ENDPOINT + "/accounts" + response = requests.get( + url, params={"limit": 100}, headers=self._get_request_headers(access_data) + ) + data = self._get_response_data(response) + for ponto_account in data.get("data", []): + ponto_iban = sanitize_account_number( + ponto_account.get("attributes", {}).get("reference", "") + ) + if ponto_iban == account_number: + access_data["ponto_account"] = ponto_account.get("id") + return + # If we get here, we did not find Ponto account for bank account. + raise UserError( + _("Ponto : wrong configuration, account %s not found in %s") + % (account_number, data) + ) + + def _ponto_synchronisation(self, access_data): + """Make sure Ponto has retrieved latest data from financial institution.""" + url = PONTO_ENDPOINT + "/synchronizations" + # TODO: According to spec we should provide an IP number in the data. + # See: https://documentation.myponto.com/1/api/curl#create-synchronization + payload = { + "data": { + "type": "synchronization", + "attributes": { + "resourceType": "account", + "resourceId": access_data["ponto_account"], + "subtype": "accountTransactions", + }, + } + } + response = requests.post( + url, headers=self._get_request_headers(access_data), json=payload + ) + if response.status_code == 400: + # Probably synchronization recently done already. + _logger.debug( + _("Syncronization request rejected: %s"), + response.text + ) + return + data = self._get_response_data(response) + sync_id = data.get("attributes", {}).get("resourceId", False) + # Check synchronisation + if not sync_id: + return + # Poll synchronization during 400 seconds for completion. + url = PONTO_ENDPOINT + "/synchronizations/" + sync_id + number = 0 + while number <= 10: + number += 1 + response = requests.get(url, headers=self._get_request_headers(access_data)) + if response.status_code == 200: + data = json.loads(response.text) + status = data.get("status", {}) + if status in ("success", "error"): + if status == "error": + _logger.debug( + _("Syncronization was succesfully completed") + ) + else: + _logger.debug( + _("Syncronization had an error: %s"), + response.text + ) + return + time.sleep(40) + + def _get_transactions(self, access_data, last_identifier): + """Get transactions from ponto, using last_identifier as pointer.""" + url = ( + PONTO_ENDPOINT + + "/accounts/" + + access_data["ponto_account"] + + "/transactions" + ) + params = {"limit": 100} + if last_identifier: + params["after"] = last_identifier + data = self._get_request(access_data, url, params) + transactions = self._get_transactions_from_data(data) + return transactions + + def _get_transactions_from_data(self, data): + """Get all transactions that are in the ponto response data.""" + transactions = data.get("data", []) + if not transactions: + _logger.debug( + _("No transactions where found in data %s"), + data, + ) + else: + _logger.debug( + _("%d transactions present in response data"), + len(transactions), + ) + return transactions + + def _get_request(self, access_data, url, params): + """Interact with Ponto to get next page of data.""" + headers = self._get_request_headers(access_data) + _logger.debug( + _("Get request to %s, with headers %s and params %s"), + url, + params, + headers + ) + response = requests.get( + url, params=params, headers=headers + ) + return self._get_response_data(response) + + def _get_response_data(self, response): + """Get response data for GET or POST request.""" + if response.status_code not in (200, 201): + raise UserError( + _("Server returned status code %s: %s") + % (response.status_code, response.text) + ) + return json.loads(response.text)