From ada0b526cb8b549408f880d02e6eb87497d6d115 Mon Sep 17 00:00:00 2001 From: Ronald Portier Date: Thu, 29 Sep 2016 15:51:31 +0200 Subject: [PATCH] [ENH] Provide alternative way to check for unique import lines. --- account_bank_statement_import/__openerp__.py | 1 + .../models/account_bank_statement_import.py | 78 ++++++++++++++----- .../models/res_partner_bank.py | 12 +++ account_bank_statement_import/parserlib.py | 50 +++++++----- .../views/res_partner_bank.xml | 22 ++++++ .../models/parser.py | 29 ++----- .../mt940.py | 22 +----- 7 files changed, 132 insertions(+), 82 deletions(-) create mode 100644 account_bank_statement_import/views/res_partner_bank.xml diff --git a/account_bank_statement_import/__openerp__.py b/account_bank_statement_import/__openerp__.py index ba892be..a952ebe 100644 --- a/account_bank_statement_import/__openerp__.py +++ b/account_bank_statement_import/__openerp__.py @@ -12,6 +12,7 @@ 'views/account_config_settings.xml', 'views/account_bank_statement_import_view.xml', 'views/account_journal.xml', + 'views/res_partner_bank.xml', ], 'demo': [ 'demo/fiscalyear_period.xml', diff --git a/account_bank_statement_import/models/account_bank_statement_import.py b/account_bank_statement_import/models/account_bank_statement_import.py index b9a1c2d..5e49d89 100644 --- a/account_bank_statement_import/models/account_bank_statement_import.py +++ b/account_bank_statement_import/models/account_bank_statement_import.py @@ -4,11 +4,13 @@ import logging import base64 from StringIO import StringIO from zipfile import ZipFile, BadZipfile # BadZipFile in Python >= 3.2 +import hashlib from openerp import api, models, fields from openerp.tools.translate import _ from openerp.exceptions import Warning as UserError, RedirectWarning + _logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -233,6 +235,16 @@ class AccountBankStatementImport(models.TransientModel): # if no currency_code is provided, we'll use the company currency return self.env.user.company_id.currency_id.id + @api.model + def _get_bank(self, account_number): + """Get res.partner.bank.""" + bank_model = self.env['res.partner.bank'] + if account_number and len(account_number) > 4: + return bank_model.search( + [('acc_number', '=', account_number)], limit=1 + ) + return bank_model.browse([]) # Empty recordset + @api.model def _find_bank_account_id(self, account_number): """ Get res.partner.bank ID """ @@ -337,12 +349,19 @@ class AccountBankStatementImport(models.TransientModel): def _complete_statement(self, stmt_vals, journal_id, account_number): """Complete statement from information passed.""" stmt_vals['journal_id'] = journal_id + statement_bank = self._get_bank(account_number) for line_vals in stmt_vals['transactions']: - unique_import_id = line_vals.get('unique_import_id', False) + unique_import_id = ( + statement_bank.enforce_unique_import_lines and + 'data' in line_vals and line_vals['data'] and + hashlib.md5(line_vals['data']).hexdigest() or + 'unique_import_id' in line_vals and + line_vals['unique_import_id'] or + False + ) if unique_import_id: line_vals['unique_import_id'] = ( - (account_number and account_number + '-' or '') + - unique_import_id + statement_bank.acc_number + '-' + unique_import_id ) if not line_vals.get('bank_account_id'): # Find the partner and his bank account or create the bank @@ -353,16 +372,15 @@ class AccountBankStatementImport(models.TransientModel): bank_account_id = False partner_account_number = line_vals.get('account_number') if partner_account_number: - bank_model = self.env['res.partner.bank'] - banks = bank_model.search( - [('acc_number', '=', partner_account_number)], limit=1) - if banks: - bank_account_id = banks[0].id - partner_id = banks[0].partner_id.id + partner_bank = self._get_bank(partner_account_number) + if partner_bank: + partner_id = partner_bank.partner_id.id else: - bank_obj = self._create_bank_account( - partner_account_number) - bank_account_id = bank_obj and bank_obj.id or False + partner_bank = self._create_bank_account( + partner_account_number + ) + if partner_bank: + bank_account_id = partner_bank.id line_vals['partner_id'] = partner_id line_vals['bank_account_id'] = bank_account_id if 'date' in stmt_vals and 'period_id' not in stmt_vals: @@ -389,22 +407,44 @@ class AccountBankStatementImport(models.TransientModel): # Filter out already imported transactions and create statement ignored_line_ids = [] filtered_st_lines = [] + unique_ids = {} + duplicates = 0 for line_vals in stmt_vals['transactions']: - unique_id = ( - 'unique_import_id' in line_vals and - line_vals['unique_import_id'] - ) - if not unique_id or not bool(bsl_model.sudo().search( - [('unique_import_id', '=', unique_id)], limit=1)): + unique_id = line_vals.get('unique_import_id', False) + if not unique_id: filtered_st_lines.append(line_vals) - else: + continue + if unique_id in unique_ids: + # Not unique within statement! + # In these cases the duplicates should both be imported. + # Most like case is if the same person made two equal payments + # on the same day. But duplicate will be marked. + duplicates += 1 + _logger.warn( + _("line with unique_id %s is not in fact unique: %s"), + unique_id, line_vals) + unique_id += " %d" % duplicates + line_vals['name'] = (_( + "Duplicate: %s") % line_vals['name'] or '') + line_vals['unique_import_id'] = unique_id + filtered_st_lines.append(line_vals) + continue + existing_line = bsl_model.sudo().search([ + ('unique_import_id', '=', unique_id), + ('company_id', '=', self.env.user.company_id.id)], limit=1) + if existing_line: ignored_line_ids.append(unique_id) + continue + unique_ids[unique_id] = line_vals + filtered_st_lines.append(line_vals) statement_id = False if len(filtered_st_lines) > 0: # Remove values that won't be used to create records stmt_vals.pop('transactions', None) for line_vals in filtered_st_lines: line_vals.pop('account_number', None) + line_vals.pop('transaction_id', None) + line_vals.pop('data', None) # Create the statement stmt_vals['line_ids'] = [ [0, False, line] for line in filtered_st_lines] diff --git a/account_bank_statement_import/models/res_partner_bank.py b/account_bank_statement_import/models/res_partner_bank.py index e3399b9..912fc6f 100644 --- a/account_bank_statement_import/models/res_partner_bank.py +++ b/account_bank_statement_import/models/res_partner_bank.py @@ -33,6 +33,18 @@ class ResPartnerBank(models.Model): sanitized_acc_number = fields.Char( 'Sanitized Account Number', size=64, readonly=True, compute='_get_sanitized_account_number', store=True, index=True) + enforce_unique_import_lines = fields.Boolean( + string='Force unique lines on import', + help="Some banks do not provide an unique id for transactions in" + " bank statements. In some cases it is possible that multiple" + " downloads contain overlapping transactions. In that case" + " activate this option to generate a unique id based on all the" + " information in the transaction. This prevents duplicate" + " imports, at the cost of - in exceptional cases - missing" + " transactions when all the information in two or more" + " transactions is the same.\n" + "This setting is only relevant for banks linked to a company." + ) def _sanitize_account_number(self, acc_number): if acc_number: diff --git a/account_bank_statement_import/parserlib.py b/account_bank_statement_import/parserlib.py index ff8012d..ee3ff6d 100644 --- a/account_bank_statement_import/parserlib.py +++ b/account_bank_statement_import/parserlib.py @@ -1,28 +1,22 @@ # -*- coding: utf-8 -*- """Classes and definitions used in parsing bank statements.""" -############################################################################## -# -# Copyright (C) 2015 Therp BV . -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## +# © 2015-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). class BankTransaction(dict): """Single transaction that is part of a bank statement.""" + @property + def transaction_id(self): + """property getter""" + return self['transaction_id'] + + @transaction_id.setter + def transaction_id(self, transaction_id): + """property setter""" + self['transaction_id'] = transaction_id + @property def value_date(self): """property getter""" @@ -106,12 +100,21 @@ class BankTransaction(dict): def note(self, note): self['note'] = note + @property + def data(self): + return self['data'] + + @data.setter + def data(self, data): + self['data'] = data + def __init__(self): """Define and initialize attributes. Not all attributes are already used in the actual import. """ super(BankTransaction, self).__init__() + self.transaction_id = '' # Fill this only if unique for import self.transfer_type = False # Action type that initiated this message self.execution_date = False # The posted date of the action self.value_date = False # The value date of the action @@ -152,8 +155,15 @@ class BankStatement(dict): subno = 0 for transaction in self['transactions']: subno += 1 - transaction['unique_import_id'] = ( - self.statement_id + str(subno).zfill(4)) + if transaction.transaction_id: + transaction['unique_import_id'] = transaction.transaction_id + else: + # Cut local_account from prefix if applicable: + if self.statement_id.startswith(self.local_account): + prefix = self.statement_id[len(self.local_account):] + else: + prefix = self.statement_id + transaction['unique_import_id'] = prefix + str(subno).zfill(4) @statement_id.setter def statement_id(self, statement_id): diff --git a/account_bank_statement_import/views/res_partner_bank.xml b/account_bank_statement_import/views/res_partner_bank.xml new file mode 100644 index 0000000..7e13a3f --- /dev/null +++ b/account_bank_statement_import/views/res_partner_bank.xml @@ -0,0 +1,22 @@ + + + + + res.partner.bank + + + + + + + + + + + diff --git a/account_bank_statement_import_camt/models/parser.py b/account_bank_statement_import_camt/models/parser.py index 0adf319..4f1c80d 100644 --- a/account_bank_statement_import_camt/models/parser.py +++ b/account_bank_statement_import_camt/models/parser.py @@ -1,37 +1,17 @@ # -*- coding: utf-8 -*- -"""Class to parse camt files.""" -############################################################################## -# -# Copyright (C) 2013-2015 Therp BV -# Copyright 2017 Open Net Sàrl -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - +# Copyright 2013-2018 Therp BV . +# Copyright 2017 Open Net Sàrl. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). import re from copy import copy from datetime import datetime from lxml import etree -from openerp import _ +from openerp import _, models from openerp.addons.account_bank_statement_import.parserlib import ( BankStatement) -from openerp import models - class CamtParser(models.AbstractModel): _name = 'account.bank.statement.import.camt.parser' @@ -177,6 +157,7 @@ class CamtParser(models.AbstractModel): details_nodes = node.xpath( './ns:NtryDtls/ns:TxDtls', namespaces={'ns': ns}) if len(details_nodes) == 0: + transaction['data'] = etree.tostring(node) yield transaction return transaction_base = transaction diff --git a/account_bank_statement_import_mt940_nl_ing/mt940.py b/account_bank_statement_import_mt940_nl_ing/mt940.py index a319e93..3ab974a 100644 --- a/account_bank_statement_import_mt940_nl_ing/mt940.py +++ b/account_bank_statement_import_mt940_nl_ing/mt940.py @@ -1,23 +1,7 @@ # -*- coding: utf-8 -*- """Implement BankStatementParser for MT940 IBAN ING files.""" -############################################################################## -# -# Copyright (C) 2014-2015 Therp BV . -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## +# © 2014-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). import re from openerp.addons.account_bank_statement_import_mt940_base.mt940 import ( MT940, str2amount, get_subfields, handle_common_subfields) @@ -48,7 +32,7 @@ class MT940Parser(MT940): self.current_transaction.transferred_amount = ( str2amount(parsed_data['sign'], parsed_data['amount'])) self.current_transaction.eref = parsed_data['reference'] - self.current_transaction.id = parsed_data['ingid'] + self.current_transaction.transaction_id = parsed_data['ingid'] def handle_tag_86(self, data): """Parse 86 tag containing reference data."""