[ENH] Enhance protection against duplicate line import.

account_bank_statement_import/__openerp__.py

@@ -12,6 +12,7 @@
         'views/account_config_settings.xml',
         'views/account_bank_statement_import_view.xml',
         'views/account_journal.xml',
+        'views/res_partner_bank.xml',
     ],
     'demo': [
         'demo/fiscalyear_period.xml',

account_bank_statement_import/models/account_bank_statement_import.py

@@ -4,11 +4,13 @@ import logging
 import base64
 from StringIO import StringIO
 from zipfile import ZipFile, BadZipfile  # BadZipFile in Python >= 3.2
+import hashlib
 
 from openerp import api, models, fields
 from openerp.tools.translate import _
 from openerp.exceptions import Warning as UserError, RedirectWarning
 
+
 _logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
 
@@ -149,13 +151,13 @@ class AccountBankStatementImport(models.TransientModel):
         account_number = stmt_vals.pop('account_number')
         # Try to find the bank account and currency in odoo
         currency_id = self._find_currency_id(currency_code)
-        bank_account_id = self._find_bank_account_id(account_number)
-        if not bank_account_id and account_number:
+        statement_bank = self._get_bank(account_number)
+        if not statement_bank and account_number:
             raise UserError(
                 _('Can not find the account number %s.') % account_number
             )
         # Find the bank journal
-        journal_id = self._get_journal(currency_id, bank_account_id)
+        journal_id = self._get_journal(currency_id, statement_bank.id)
         # By now journal and account_number must be known
         if not journal_id:
             raise UserError(
@@ -165,7 +167,8 @@ class AccountBankStatementImport(models.TransientModel):
             )
         # Prepare statement data to be used for bank statements creation
         stmt_vals = self._complete_statement(
-            stmt_vals, journal_id, account_number)
+            statement_bank, stmt_vals, journal_id
+        )
         # Create the bank stmt_vals
         return self._create_bank_statement(stmt_vals)
 
@@ -234,15 +237,14 @@ class AccountBankStatementImport(models.TransientModel):
         return self.env.user.company_id.currency_id.id
 
     @api.model
-    def _find_bank_account_id(self, account_number):
-        """ Get res.partner.bank ID """
-        bank_account_id = None
+    def _get_bank(self, account_number):
+        """Get res.partner.bank."""
+        bank_model = self.env['res.partner.bank']
         if account_number and len(account_number) > 4:
-            bank_account_ids = self.env['res.partner.bank'].search(
-                [('acc_number', '=', account_number)], limit=1)
-            if bank_account_ids:
-                bank_account_id = bank_account_ids[0].id
-        return bank_account_id
+            return bank_model.search(
+                [('acc_number', '=', account_number)], limit=1
+            )
+        return bank_model.browse([])  # Empty recordset
 
     @api.model
     def _get_journal(self, currency_id, bank_account_id):
@@ -334,15 +336,21 @@ class AccountBankStatementImport(models.TransientModel):
             default_currency=currency_id).create(vals_acc)
 
     @api.model
-    def _complete_statement(self, stmt_vals, journal_id, account_number):
+    def _complete_statement(self, statement_bank, stmt_vals, journal_id):
         """Complete statement from information passed."""
         stmt_vals['journal_id'] = journal_id
         for line_vals in stmt_vals['transactions']:
-            unique_import_id = line_vals.get('unique_import_id', False)
+            unique_import_id = (
+                statement_bank.enforce_unique_import_lines and
+                'data' in line_vals and line_vals['data'] and
+                hashlib.md5(line_vals['data']) or
+                'unique_import_id' in line_vals and
+                line_vals['unique_import_id'] or
+                False
+            )
             if unique_import_id:
                 line_vals['unique_import_id'] = (
-                    (account_number and account_number + '-' or '') +
-                    unique_import_id
+                    statement_bank.acc_number + '-' + unique_import_id
                 )
             if not line_vals.get('bank_account_id'):
                 # Find the partner and his bank account or create the bank
@@ -353,16 +361,15 @@ class AccountBankStatementImport(models.TransientModel):
                 bank_account_id = False
                 partner_account_number = line_vals.get('account_number')
                 if partner_account_number:
-                    bank_model = self.env['res.partner.bank']
-                    banks = bank_model.search(
-                        [('acc_number', '=', partner_account_number)], limit=1)
-                    if banks:
-                        bank_account_id = banks[0].id
-                        partner_id = banks[0].partner_id.id
+                    partner_bank = self._get_bank(partner_account_number)
+                    if partner_bank:
+                        partner_id = partner_bank.partner_id.id
                     else:
-                        bank_obj = self._create_bank_account(
-                            partner_account_number)
-                        bank_account_id = bank_obj and bank_obj.id or False
+                        partner_bank = self._create_bank_account(
+                            partner_account_number
+                        )
+                    if partner_bank:
+                        bank_account_id = partner_bank.id
                 line_vals['partner_id'] = partner_id
                 line_vals['bank_account_id'] = bank_account_id
         if 'date' in stmt_vals and 'period_id' not in stmt_vals:
@@ -405,6 +412,8 @@ class AccountBankStatementImport(models.TransientModel):
             stmt_vals.pop('transactions', None)
             for line_vals in filtered_st_lines:
                 line_vals.pop('account_number', None)
+                line_vals.pop('statement_id', None)
+                line_vals.pop('data', None)
             # Create the statement
             stmt_vals['line_ids'] = [
                 [0, False, line] for line in filtered_st_lines]

account_bank_statement_import/models/res_partner_bank.py

@@ -33,6 +33,18 @@ class ResPartnerBank(models.Model):
     sanitized_acc_number = fields.Char(
         'Sanitized Account Number', size=64, readonly=True,
         compute='_get_sanitized_account_number', store=True, index=True)
+    enforce_unique_import_lines = fields.Boolean(
+        string='Force unique lines on import',
+        help="Some banks do not provide an unique id for transactions in"
+             " bank statements. In some cases it is possible that multiple"
+             " downloads contain overlapping transactions. In that case"
+             " activate this option to generate a unique id based on all the"
+             " information in the transaction. This prevents duplicate"
+             " imports, at the cost of - in exceptional cases - missing"
+             " transactions when all the information in two or more"
+             " transactions is the same.\n"
+             "This setting is only relevant for banks linked to a company."
+    )
 
     def _sanitize_account_number(self, acc_number):
         if acc_number:

account_bank_statement_import/parserlib.py

@@ -100,6 +100,14 @@ class BankTransaction(dict):
     def note(self, note):
         self['note'] = note
 
+    @property
+    def data(self):
+        return self['data']
+
+    @data.setter
+    def data(self, data):
+        self['data'] = data
+
     def __init__(self):
         """Define and initialize attributes.
 

account_bank_statement_import/views/res_partner_bank.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<openerp>
+    <data>
+        <record id="view_partner_bank_form" model="ir.ui.view">
+            <field name="model">res.partner.bank</field>
+            <field name="inherit_id" ref="base.view_partner_bank_form" />
+            <field name="arch" type="xml">
+                <group name="bank" position="after">
+                    <group
+                        name="import_settings"
+                        string="Bank Statement Import Settings"
+                        invisible="context.get('company_hide', True)"
+                        >
+                        <field
+                            name="enforce_unique_import_lines"
+                        />
+                    </group>
+                </group>
+            </field>
+        </record>
+    </data>
+</openerp>

account_bank_statement_import_camt/models/parser.py

@@ -1,25 +1,8 @@
 # -*- coding: utf-8 -*-
-"""Class to parse camt files."""
-##############################################################################
-#
-#    Copyright (C) 2013-2018 Therp BV <http://therp.nl>
+# Copyright 2013-2018 Therp BV <http://therp.nl>
 # Copyright 2017 Open Net Sàrl
-#              (C) 2015 1200wd.com
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as published
-#    by the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
+# Copyright 2015 1200wd.com
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 import logging
 import re
 from copy import copy
@@ -263,6 +246,7 @@ class CamtParser(models.AbstractModel):
         for entry_node in transaction_nodes:
             transaction = statement.create_transaction()
             total_amount += transaction['transferred_amount']
+            transaction.data = etree.tostring(entry_node)
             self.parse_transaction(ns, entry_node, transaction)
         if statement['transactions']:
             execution_date = statement['transactions'][0].execution_date[:10]