From e38d424e46d8ffa83c87c7cbf754f3d957ecb177 Mon Sep 17 00:00:00 2001 From: Ronald Portier Date: Fri, 14 Oct 2016 15:42:26 +0200 Subject: [PATCH] [ENH] Enhance protection against duplicate line import. --- account_bank_statement_import/__openerp__.py | 1 + .../models/account_bank_statement_import.py | 59 +++++++++++-------- .../models/res_partner_bank.py | 12 ++++ account_bank_statement_import/parserlib.py | 8 +++ .../views/res_partner_bank.xml | 22 +++++++ .../models/parser.py | 26 ++------ 6 files changed, 82 insertions(+), 46 deletions(-) create mode 100644 account_bank_statement_import/views/res_partner_bank.xml diff --git a/account_bank_statement_import/__openerp__.py b/account_bank_statement_import/__openerp__.py index ba892be..a952ebe 100644 --- a/account_bank_statement_import/__openerp__.py +++ b/account_bank_statement_import/__openerp__.py @@ -12,6 +12,7 @@ 'views/account_config_settings.xml', 'views/account_bank_statement_import_view.xml', 'views/account_journal.xml', + 'views/res_partner_bank.xml', ], 'demo': [ 'demo/fiscalyear_period.xml', diff --git a/account_bank_statement_import/models/account_bank_statement_import.py b/account_bank_statement_import/models/account_bank_statement_import.py index b9a1c2d..94f88c1 100644 --- a/account_bank_statement_import/models/account_bank_statement_import.py +++ b/account_bank_statement_import/models/account_bank_statement_import.py @@ -4,11 +4,13 @@ import logging import base64 from StringIO import StringIO from zipfile import ZipFile, BadZipfile # BadZipFile in Python >= 3.2 +import hashlib from openerp import api, models, fields from openerp.tools.translate import _ from openerp.exceptions import Warning as UserError, RedirectWarning + _logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -149,13 +151,13 @@ class AccountBankStatementImport(models.TransientModel): account_number = stmt_vals.pop('account_number') # Try to find the bank account and currency in odoo currency_id = self._find_currency_id(currency_code) - bank_account_id = self._find_bank_account_id(account_number) - if not bank_account_id and account_number: + statement_bank = self._get_bank(account_number) + if not statement_bank and account_number: raise UserError( _('Can not find the account number %s.') % account_number ) # Find the bank journal - journal_id = self._get_journal(currency_id, bank_account_id) + journal_id = self._get_journal(currency_id, statement_bank.id) # By now journal and account_number must be known if not journal_id: raise UserError( @@ -165,7 +167,8 @@ class AccountBankStatementImport(models.TransientModel): ) # Prepare statement data to be used for bank statements creation stmt_vals = self._complete_statement( - stmt_vals, journal_id, account_number) + statement_bank, stmt_vals, journal_id + ) # Create the bank stmt_vals return self._create_bank_statement(stmt_vals) @@ -234,15 +237,14 @@ class AccountBankStatementImport(models.TransientModel): return self.env.user.company_id.currency_id.id @api.model - def _find_bank_account_id(self, account_number): - """ Get res.partner.bank ID """ - bank_account_id = None + def _get_bank(self, account_number): + """Get res.partner.bank.""" + bank_model = self.env['res.partner.bank'] if account_number and len(account_number) > 4: - bank_account_ids = self.env['res.partner.bank'].search( - [('acc_number', '=', account_number)], limit=1) - if bank_account_ids: - bank_account_id = bank_account_ids[0].id - return bank_account_id + return bank_model.search( + [('acc_number', '=', account_number)], limit=1 + ) + return bank_model.browse([]) # Empty recordset @api.model def _get_journal(self, currency_id, bank_account_id): @@ -334,15 +336,21 @@ class AccountBankStatementImport(models.TransientModel): default_currency=currency_id).create(vals_acc) @api.model - def _complete_statement(self, stmt_vals, journal_id, account_number): + def _complete_statement(self, statement_bank, stmt_vals, journal_id): """Complete statement from information passed.""" stmt_vals['journal_id'] = journal_id for line_vals in stmt_vals['transactions']: - unique_import_id = line_vals.get('unique_import_id', False) + unique_import_id = ( + statement_bank.enforce_unique_import_lines and + 'data' in line_vals and line_vals['data'] and + hashlib.md5(line_vals['data']) or + 'unique_import_id' in line_vals and + line_vals['unique_import_id'] or + False + ) if unique_import_id: line_vals['unique_import_id'] = ( - (account_number and account_number + '-' or '') + - unique_import_id + statement_bank.acc_number + '-' + unique_import_id ) if not line_vals.get('bank_account_id'): # Find the partner and his bank account or create the bank @@ -353,16 +361,15 @@ class AccountBankStatementImport(models.TransientModel): bank_account_id = False partner_account_number = line_vals.get('account_number') if partner_account_number: - bank_model = self.env['res.partner.bank'] - banks = bank_model.search( - [('acc_number', '=', partner_account_number)], limit=1) - if banks: - bank_account_id = banks[0].id - partner_id = banks[0].partner_id.id + partner_bank = self._get_bank(partner_account_number) + if partner_bank: + partner_id = partner_bank.partner_id.id else: - bank_obj = self._create_bank_account( - partner_account_number) - bank_account_id = bank_obj and bank_obj.id or False + partner_bank = self._create_bank_account( + partner_account_number + ) + if partner_bank: + bank_account_id = partner_bank.id line_vals['partner_id'] = partner_id line_vals['bank_account_id'] = bank_account_id if 'date' in stmt_vals and 'period_id' not in stmt_vals: @@ -405,6 +412,8 @@ class AccountBankStatementImport(models.TransientModel): stmt_vals.pop('transactions', None) for line_vals in filtered_st_lines: line_vals.pop('account_number', None) + line_vals.pop('statement_id', None) + line_vals.pop('data', None) # Create the statement stmt_vals['line_ids'] = [ [0, False, line] for line in filtered_st_lines] diff --git a/account_bank_statement_import/models/res_partner_bank.py b/account_bank_statement_import/models/res_partner_bank.py index e3399b9..912fc6f 100644 --- a/account_bank_statement_import/models/res_partner_bank.py +++ b/account_bank_statement_import/models/res_partner_bank.py @@ -33,6 +33,18 @@ class ResPartnerBank(models.Model): sanitized_acc_number = fields.Char( 'Sanitized Account Number', size=64, readonly=True, compute='_get_sanitized_account_number', store=True, index=True) + enforce_unique_import_lines = fields.Boolean( + string='Force unique lines on import', + help="Some banks do not provide an unique id for transactions in" + " bank statements. In some cases it is possible that multiple" + " downloads contain overlapping transactions. In that case" + " activate this option to generate a unique id based on all the" + " information in the transaction. This prevents duplicate" + " imports, at the cost of - in exceptional cases - missing" + " transactions when all the information in two or more" + " transactions is the same.\n" + "This setting is only relevant for banks linked to a company." + ) def _sanitize_account_number(self, acc_number): if acc_number: diff --git a/account_bank_statement_import/parserlib.py b/account_bank_statement_import/parserlib.py index 7025bb1..ee3ff6d 100644 --- a/account_bank_statement_import/parserlib.py +++ b/account_bank_statement_import/parserlib.py @@ -100,6 +100,14 @@ class BankTransaction(dict): def note(self, note): self['note'] = note + @property + def data(self): + return self['data'] + + @data.setter + def data(self, data): + self['data'] = data + def __init__(self): """Define and initialize attributes. diff --git a/account_bank_statement_import/views/res_partner_bank.xml b/account_bank_statement_import/views/res_partner_bank.xml new file mode 100644 index 0000000..968c4bf --- /dev/null +++ b/account_bank_statement_import/views/res_partner_bank.xml @@ -0,0 +1,22 @@ + + + + + res.partner.bank + + + + + + + + + + + diff --git a/account_bank_statement_import_camt/models/parser.py b/account_bank_statement_import_camt/models/parser.py index 3f8b957..b531e06 100644 --- a/account_bank_statement_import_camt/models/parser.py +++ b/account_bank_statement_import_camt/models/parser.py @@ -1,25 +1,8 @@ # -*- coding: utf-8 -*- -"""Class to parse camt files.""" -############################################################################## -# -# Copyright (C) 2013-2018 Therp BV -# Copyright 2017 Open Net Sàrl -# (C) 2015 1200wd.com -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## +# Copyright 2013-2018 Therp BV +# Copyright 2017 Open Net Sàrl +# Copyright 2015 1200wd.com +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). import logging import re from copy import copy @@ -263,6 +246,7 @@ class CamtParser(models.AbstractModel): for entry_node in transaction_nodes: transaction = statement.create_transaction() total_amount += transaction['transferred_amount'] + transaction.data = etree.tostring(entry_node) self.parse_transaction(ns, entry_node, transaction) if statement['transactions']: execution_date = statement['transactions'][0].execution_date[:10]