Browse Source
[IMP] *_online_ponto: more efficient ponto account retrieval
[IMP] *_online_ponto: more efficient ponto account retrieval
Also do not name variables account_id or account_ids when they do not refer to ids in the Odoo sense, nor to either bank or ledger accounts.12.0
Ronald Portier (Therp BV)
2 years ago
No known key found for this signature in database
GPG Key ID: A181F8124D7101D3
6 changed files with 360 additions and 211 deletions
-
8account_bank_statement_import_online_ponto/__manifest__.py
-
5account_bank_statement_import_online_ponto/models/__init__.py
-
261account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
-
79account_bank_statement_import_online_ponto/models/ponto_buffer.py
-
28account_bank_statement_import_online_ponto/models/ponto_buffer_line.py
-
190account_bank_statement_import_online_ponto/models/ponto_interface.py
@ -1,4 +1,5 @@ |
|||
# Copyright 2020 Florent de Labarre |
|||
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). |
|||
|
|||
from . import ponto_buffer |
|||
from . import ponto_buffer_line |
|||
from . import ponto_interface |
|||
from . import online_bank_statement_provider_ponto |
|||
@ -0,0 +1,79 @@ |
|||
# Copyright 2022 Therp BV <https://therp.nl>. |
|||
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). |
|||
"""Define model to hold data retrieved from Ponto.""" |
|||
import json |
|||
import logging |
|||
|
|||
from odoo import _, fields, models |
|||
|
|||
_logger = logging.getLogger(__name__) |
|||
|
|||
|
|||
class PontoBuffer(models.Model): |
|||
"""Define model to hold data retrieved from Ponto.""" |
|||
_name = "ponto.buffer" |
|||
_description = "Group transactions retrieved from Ponto." |
|||
|
|||
provider_id = fields.Many2one( |
|||
comodel_name="online.bank.statement.provider", |
|||
required=True, |
|||
readonly=True, |
|||
) |
|||
effective_date = fields.Date(readonly=True, required=True) |
|||
buffer_line_ids = fields.One2many( |
|||
comodel_name="ponto.buffer.line", |
|||
inverse_name="buffer_id", |
|||
readonly=True, |
|||
ondelete="cascade", |
|||
) |
|||
|
|||
def _store_transactions(self, provider, transactions): |
|||
"""Store transactions retrieved from Ponto in buffer, preventing duplicates.""" |
|||
# Start by sorting all transactions per date. |
|||
transactions_per_date = {} |
|||
for transaction in transactions: |
|||
ponto_execution_date = transaction.get( |
|||
"attributes", {} |
|||
).get("executionDate") |
|||
effective_date_time = provider._ponto_date_from_string(ponto_execution_date) |
|||
transaction["effective_date_time"] = effective_date_time.isoformat() |
|||
key = effective_date_time.isoformat()[0:10] |
|||
if key not in transactions_per_date: |
|||
transactions_per_date[key] = [] # Initialize transaction array. |
|||
transactions_per_date[key].append(transaction) |
|||
# Now store the transactions, but not when already present. |
|||
for key, date_transactions in transactions_per_date.items(): |
|||
_logger.debug( |
|||
_("For date %s we retrieved %d transactions"), |
|||
key, |
|||
len(date_transactions) |
|||
) |
|||
ponto_buffer = self.search( |
|||
[ |
|||
("provider_id", "=", provider.id), |
|||
("effective_date", "=", key), |
|||
], |
|||
limit=1 |
|||
) or self.create( |
|||
{ |
|||
"provider_id": provider.id, |
|||
"effective_date": key, |
|||
} |
|||
) |
|||
already_present = ponto_buffer.buffer_line_ids.mapped("ponto_id") |
|||
new_lines = [ |
|||
( |
|||
0, |
|||
0, |
|||
{ |
|||
"buffer_id": ponto_buffer.id, |
|||
"ponto_id": t.get("id"), |
|||
"effective_date_time": t.get("effective_date_time"), |
|||
"transaction_data": json.dumps(t), |
|||
}, |
|||
) |
|||
for t in date_transactions |
|||
if t.get("id") not in already_present |
|||
] |
|||
if new_lines: |
|||
ponto_buffer.write({"buffer_line_ids": new_lines}) |
|||
@ -0,0 +1,28 @@ |
|||
# Copyright 2022 Therp BV <https://therp.nl>. |
|||
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). |
|||
"""Define model to hold transactions retrieved from Ponto.""" |
|||
from odoo import fields, models |
|||
|
|||
|
|||
class PontoBuffer(models.Model): |
|||
"""Define model to hold transactions retrieved from Ponto.""" |
|||
_name = "ponto.buffer.line" |
|||
_description = "Hold transactions retrieved from Ponto." |
|||
|
|||
buffer_id = fields.Many2one( |
|||
comodel_name="ponto.buffer", |
|||
required=True, |
|||
readonly=True, |
|||
) |
|||
ponto_id = fields.Char( |
|||
required=True, |
|||
readonly=True, |
|||
) |
|||
effective_date_time = fields.Datetime( |
|||
required=True, |
|||
readonly=True, |
|||
) |
|||
transaction_data = fields.Char( |
|||
required=True, |
|||
readonly=True, |
|||
) |
|||
@ -0,0 +1,190 @@ |
|||
# Copyright 2020 Florent de Labarre |
|||
# Copyright 2022 Therp BV <https://therp.nl>. |
|||
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). |
|||
|
|||
import base64 |
|||
import json |
|||
import logging |
|||
import time |
|||
|
|||
import requests |
|||
from dateutil.relativedelta import relativedelta |
|||
|
|||
from odoo import fields, models, _ |
|||
from odoo.exceptions import UserError |
|||
from odoo.addons.base.models.res_bank import sanitize_account_number |
|||
|
|||
_logger = logging.getLogger(__name__) |
|||
|
|||
PONTO_ENDPOINT = "https://api.myponto.com" |
|||
|
|||
|
|||
class PontoInterface(models.AbstractModel): |
|||
_name = "ponto.interface" |
|||
_description = "Interface to all interactions with Ponto API" |
|||
|
|||
def _login(self, username, password): |
|||
"""Ponto login returns an access dictionary for further requests.""" |
|||
url = PONTO_ENDPOINT + "/oauth2/token" |
|||
if not(username and password): |
|||
raise UserError(_("Please fill login and key.")) |
|||
login = "%s:%s" % (username, password) |
|||
login = base64.b64encode(login.encode("UTF-8")).decode("UTF-8") |
|||
login_headers = { |
|||
"Content-Type": "application/x-www-form-urlencoded", |
|||
"Accept": "application/json", |
|||
"Authorization": "Basic %s" % login, |
|||
} |
|||
response = requests.post( |
|||
url, |
|||
params={"grant_type": "client_credentials"}, |
|||
headers=login_headers, |
|||
) |
|||
data = self._get_response_data(response) |
|||
access_token = data.get("access_token", False) |
|||
if not access_token: |
|||
raise UserError(_("Ponto : no token")) |
|||
token_expiration = fields.Datetime.now() + relativedelta( |
|||
seconds=data.get("expires_in", False) |
|||
) |
|||
return { |
|||
"username": username, |
|||
"password": password, |
|||
"access_token": access_token, |
|||
"token_expiration": token_expiration, |
|||
} |
|||
|
|||
def _get_request_headers(self, access_data): |
|||
"""Get headers with authorization for further ponto requests.""" |
|||
if access_data["token_expiration"] <= fields.Datetime.now(): |
|||
updated_data = self._login(access_data["username"], access_data["password"]) |
|||
access_data.update(updated_data) |
|||
return { |
|||
"Accept": "application/json", |
|||
"Authorization": "Bearer %s" % access_data["access_token"], |
|||
} |
|||
|
|||
def _set_access_account(self, access_data, account_number): |
|||
"""Set ponto account for bank account in access_data.""" |
|||
url = PONTO_ENDPOINT + "/accounts" |
|||
response = requests.get( |
|||
url, params={"limit": 100}, headers=self._get_request_headers(access_data) |
|||
) |
|||
data = self._get_response_data(response) |
|||
for ponto_account in data.get("data", []): |
|||
ponto_iban = sanitize_account_number( |
|||
ponto_account.get("attributes", {}).get("reference", "") |
|||
) |
|||
if ponto_iban == account_number: |
|||
access_data["ponto_account"] = ponto_account.get("id") |
|||
return |
|||
# If we get here, we did not find Ponto account for bank account. |
|||
raise UserError( |
|||
_("Ponto : wrong configuration, account %s not found in %s") |
|||
% (account_number, data) |
|||
) |
|||
|
|||
def _ponto_synchronisation(self, access_data): |
|||
"""Make sure Ponto has retrieved latest data from financial institution.""" |
|||
url = PONTO_ENDPOINT + "/synchronizations" |
|||
# TODO: According to spec we should provide an IP number in the data. |
|||
# See: https://documentation.myponto.com/1/api/curl#create-synchronization |
|||
payload = { |
|||
"data": { |
|||
"type": "synchronization", |
|||
"attributes": { |
|||
"resourceType": "account", |
|||
"resourceId": access_data["ponto_account"], |
|||
"subtype": "accountTransactions", |
|||
}, |
|||
} |
|||
} |
|||
response = requests.post( |
|||
url, headers=self._get_request_headers(access_data), json=payload |
|||
) |
|||
if response.status_code == 400: |
|||
# Probably synchronization recently done already. |
|||
_logger.debug( |
|||
_("Syncronization request rejected: %s"), |
|||
response.text |
|||
) |
|||
return |
|||
data = self._get_response_data(response) |
|||
sync_id = data.get("attributes", {}).get("resourceId", False) |
|||
# Check synchronisation |
|||
if not sync_id: |
|||
return |
|||
# Poll synchronization during 400 seconds for completion. |
|||
url = PONTO_ENDPOINT + "/synchronizations/" + sync_id |
|||
number = 0 |
|||
while number <= 10: |
|||
number += 1 |
|||
response = requests.get(url, headers=self._get_request_headers(access_data)) |
|||
if response.status_code == 200: |
|||
data = json.loads(response.text) |
|||
status = data.get("status", {}) |
|||
if status in ("success", "error"): |
|||
if status == "error": |
|||
_logger.debug( |
|||
_("Syncronization was succesfully completed") |
|||
) |
|||
else: |
|||
_logger.debug( |
|||
_("Syncronization had an error: %s"), |
|||
response.text |
|||
) |
|||
return |
|||
time.sleep(40) |
|||
|
|||
def _get_transactions(self, access_data, last_identifier): |
|||
"""Get transactions from ponto, using last_identifier as pointer.""" |
|||
url = ( |
|||
PONTO_ENDPOINT |
|||
+ "/accounts/" |
|||
+ access_data["ponto_account"] |
|||
+ "/transactions" |
|||
) |
|||
params = {"limit": 100} |
|||
if last_identifier: |
|||
params["after"] = last_identifier |
|||
data = self._get_request(access_data, url, params) |
|||
transactions = self._get_transactions_from_data(data) |
|||
return transactions |
|||
|
|||
def _get_transactions_from_data(self, data): |
|||
"""Get all transactions that are in the ponto response data.""" |
|||
transactions = data.get("data", []) |
|||
if not transactions: |
|||
_logger.debug( |
|||
_("No transactions where found in data %s"), |
|||
data, |
|||
) |
|||
else: |
|||
_logger.debug( |
|||
_("%d transactions present in response data"), |
|||
len(transactions), |
|||
) |
|||
return transactions |
|||
|
|||
def _get_request(self, access_data, url, params): |
|||
"""Interact with Ponto to get next page of data.""" |
|||
headers = self._get_request_headers(access_data) |
|||
_logger.debug( |
|||
_("Get request to %s, with headers %s and params %s"), |
|||
url, |
|||
params, |
|||
headers |
|||
) |
|||
response = requests.get( |
|||
url, params=params, headers=headers |
|||
) |
|||
return self._get_response_data(response) |
|||
|
|||
def _get_response_data(self, response): |
|||
"""Get response data for GET or POST request.""" |
|||
if response.status_code not in (200, 201): |
|||
raise UserError( |
|||
_("Server returned status code %s: %s") |
|||
% (response.status_code, response.text) |
|||
) |
|||
return json.loads(response.text) |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue