[ENH] Provide alternative way to check for unique import lines.

account_bank_statement_import/__openerp__.py

@@ -12,6 +12,7 @@
         'views/account_config_settings.xml',
         'views/account_bank_statement_import_view.xml',
         'views/account_journal.xml',
+        'views/res_partner_bank.xml',
     ],
     'demo': [
         'demo/fiscalyear_period.xml',

account_bank_statement_import/models/account_bank_statement_import.py

@@ -4,11 +4,13 @@ import logging
 import base64
 from StringIO import StringIO
 from zipfile import ZipFile, BadZipfile  # BadZipFile in Python >= 3.2
+import hashlib
 
 from openerp import api, models, fields
 from openerp.tools.translate import _
 from openerp.exceptions import Warning as UserError, RedirectWarning
 
+
 _logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
 
@@ -233,6 +235,16 @@ class AccountBankStatementImport(models.TransientModel):
         # if no currency_code is provided, we'll use the company currency
         return self.env.user.company_id.currency_id.id
 
+    @api.model
+    def _get_bank(self, account_number):
+        """Get res.partner.bank."""
+        bank_model = self.env['res.partner.bank']
+        if account_number and len(account_number) > 4:
+            return bank_model.search(
+                [('acc_number', '=', account_number)], limit=1
+            )
+        return bank_model.browse([])  # Empty recordset
+
     @api.model
     def _find_bank_account_id(self, account_number):
         """ Get res.partner.bank ID """
@@ -337,12 +349,19 @@ class AccountBankStatementImport(models.TransientModel):
     def _complete_statement(self, stmt_vals, journal_id, account_number):
         """Complete statement from information passed."""
         stmt_vals['journal_id'] = journal_id
+        statement_bank = self._get_bank(account_number)
         for line_vals in stmt_vals['transactions']:
-            unique_import_id = line_vals.get('unique_import_id', False)
+            unique_import_id = (
+                statement_bank.enforce_unique_import_lines and
+                'data' in line_vals and line_vals['data'] and
+                hashlib.md5(line_vals['data']).hexdigest() or
+                'unique_import_id' in line_vals and
+                line_vals['unique_import_id'] or
+                False
+            )
             if unique_import_id:
                 line_vals['unique_import_id'] = (
-                    (account_number and account_number + '-' or '') +
-                    unique_import_id
+                    statement_bank.acc_number + '-' + unique_import_id
                 )
             if not line_vals.get('bank_account_id'):
                 # Find the partner and his bank account or create the bank
@@ -353,16 +372,15 @@ class AccountBankStatementImport(models.TransientModel):
                 bank_account_id = False
                 partner_account_number = line_vals.get('account_number')
                 if partner_account_number:
-                    bank_model = self.env['res.partner.bank']
-                    banks = bank_model.search(
-                        [('acc_number', '=', partner_account_number)], limit=1)
-                    if banks:
-                        bank_account_id = banks[0].id
-                        partner_id = banks[0].partner_id.id
+                    partner_bank = self._get_bank(partner_account_number)
+                    if partner_bank:
+                        partner_id = partner_bank.partner_id.id
                     else:
-                        bank_obj = self._create_bank_account(
-                            partner_account_number)
-                        bank_account_id = bank_obj and bank_obj.id or False
+                        partner_bank = self._create_bank_account(
+                            partner_account_number
+                        )
+                    if partner_bank:
+                        bank_account_id = partner_bank.id
                 line_vals['partner_id'] = partner_id
                 line_vals['bank_account_id'] = bank_account_id
         if 'date' in stmt_vals and 'period_id' not in stmt_vals:
@@ -389,22 +407,44 @@ class AccountBankStatementImport(models.TransientModel):
         # Filter out already imported transactions and create statement
         ignored_line_ids = []
         filtered_st_lines = []
+        unique_ids = {}
+        duplicates = 0
         for line_vals in stmt_vals['transactions']:
-            unique_id = (
-                'unique_import_id' in line_vals and
-                line_vals['unique_import_id']
-            )
-            if not unique_id or not bool(bsl_model.sudo().search(
-                    [('unique_import_id', '=', unique_id)], limit=1)):
+            unique_id = line_vals.get('unique_import_id', False)
+            if not unique_id:
                 filtered_st_lines.append(line_vals)
-            else:
+                continue
+            if unique_id in unique_ids:
+                # Not unique within statement!
+                # In these cases the duplicates should both be imported.
+                # Most like case is if the same person made two equal payments
+                # on the same day. But duplicate will be marked.
+                duplicates += 1
+                _logger.warn(
+                    _("line with unique_id %s is not in fact unique: %s"),
+                    unique_id, line_vals)
+                unique_id += " %d" % duplicates
+                line_vals['name'] = (_(
+                    "Duplicate: %s") % line_vals['name'] or '')
+                line_vals['unique_import_id'] = unique_id
+                filtered_st_lines.append(line_vals)
+                continue
+            existing_line = bsl_model.sudo().search([
+                ('unique_import_id', '=', unique_id),
+                ('company_id', '=', self.env.user.company_id.id)], limit=1)
+            if existing_line:
                 ignored_line_ids.append(unique_id)
+                continue
+            unique_ids[unique_id] = line_vals
+            filtered_st_lines.append(line_vals)
         statement_id = False
         if len(filtered_st_lines) > 0:
             # Remove values that won't be used to create records
             stmt_vals.pop('transactions', None)
             for line_vals in filtered_st_lines:
                 line_vals.pop('account_number', None)
+                line_vals.pop('transaction_id', None)
+                line_vals.pop('data', None)
             # Create the statement
             stmt_vals['line_ids'] = [
                 [0, False, line] for line in filtered_st_lines]

account_bank_statement_import/models/res_partner_bank.py

@@ -33,6 +33,18 @@ class ResPartnerBank(models.Model):
     sanitized_acc_number = fields.Char(
         'Sanitized Account Number', size=64, readonly=True,
         compute='_get_sanitized_account_number', store=True, index=True)
+    enforce_unique_import_lines = fields.Boolean(
+        string='Force unique lines on import',
+        help="Some banks do not provide an unique id for transactions in"
+             " bank statements. In some cases it is possible that multiple"
+             " downloads contain overlapping transactions. In that case"
+             " activate this option to generate a unique id based on all the"
+             " information in the transaction. This prevents duplicate"
+             " imports, at the cost of - in exceptional cases - missing"
+             " transactions when all the information in two or more"
+             " transactions is the same.\n"
+             "This setting is only relevant for banks linked to a company."
+    )
 
     def _sanitize_account_number(self, acc_number):
         if acc_number:

account_bank_statement_import/parserlib.py

@@ -1,28 +1,22 @@
 # -*- coding: utf-8 -*-
 """Classes and definitions used in parsing bank statements."""
-##############################################################################
-#
-#  Copyright (C) 2015 Therp BV <http://therp.nl>.
-#
-#  This program is free software: you can redistribute it and/or modify
-#  it under the terms of the GNU Affero General Public License as published by
-#  the Free Software Foundation, either version 3 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU Affero General Public License for more details.
-#
-#  You should have received a copy of the GNU Affero General Public License
-#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
+# © 2015-2016 Therp BV <http://therp.nl>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 
 
 class BankTransaction(dict):
     """Single transaction that is part of a bank statement."""
 
+    @property
+    def transaction_id(self):
+        """property getter"""
+        return self['transaction_id']
+
+    @transaction_id.setter
+    def transaction_id(self, transaction_id):
+        """property setter"""
+        self['transaction_id'] = transaction_id
+
     @property
     def value_date(self):
         """property getter"""
@@ -106,12 +100,21 @@ class BankTransaction(dict):
     def note(self, note):
         self['note'] = note
 
+    @property
+    def data(self):
+        return self['data']
+
+    @data.setter
+    def data(self, data):
+        self['data'] = data
+
     def __init__(self):
         """Define and initialize attributes.
 
         Not all attributes are already used in the actual import.
         """
         super(BankTransaction, self).__init__()
+        self.transaction_id = ''  # Fill this only if unique for import
         self.transfer_type = False  # Action type that initiated this message
         self.execution_date = False  # The posted date of the action
         self.value_date = False  # The value date of the action
@@ -152,8 +155,15 @@ class BankStatement(dict):
         subno = 0
         for transaction in self['transactions']:
             subno += 1
-            transaction['unique_import_id'] = (
-                self.statement_id + str(subno).zfill(4))
+            if transaction.transaction_id:
+                transaction['unique_import_id'] = transaction.transaction_id
+            else:
+                # Cut local_account from prefix if applicable:
+                if self.statement_id.startswith(self.local_account):
+                    prefix = self.statement_id[len(self.local_account):]
+                else:
+                    prefix = self.statement_id
+                transaction['unique_import_id'] = prefix + str(subno).zfill(4)
 
     @statement_id.setter
     def statement_id(self, statement_id):

account_bank_statement_import/views/res_partner_bank.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<openerp>
+    <data>
+        <record id="view_partner_bank_form" model="ir.ui.view">
+            <field name="model">res.partner.bank</field>
+            <field name="inherit_id" ref="base.view_partner_bank_form" />
+            <field name="arch" type="xml">
+                <group name="bank" position="after">
+                    <group
+                        name="import_settings"
+                        string="Bank Statement Import Settings"
+                        invisible="context.get('company_hide', False)"
+                        >
+                        <field
+                            name="enforce_unique_import_lines"
+                        />
+                    </group>
+                </group>
+            </field>
+        </record>
+    </data>
+</openerp>

account_bank_statement_import_camt/models/parser.py

@@ -1,37 +1,17 @@
 # -*- coding: utf-8 -*-
-"""Class to parse camt files."""
-##############################################################################
-#
-#    Copyright (C) 2013-2015 Therp BV <http://therp.nl>
-#    Copyright 2017 Open Net Sàrl
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as published
-#    by the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
-
+# Copyright 2013-2018 Therp BV <https://therp.nl>.
+# Copyright 2017 Open Net Sàrl.
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 import re
 from copy import copy
 from datetime import datetime
 from lxml import etree
 
-from openerp import _
+from openerp import _, models
 
 from openerp.addons.account_bank_statement_import.parserlib import (
     BankStatement)
 
-from openerp import models
-
 
 class CamtParser(models.AbstractModel):
     _name = 'account.bank.statement.import.camt.parser'
@@ -177,6 +157,7 @@ class CamtParser(models.AbstractModel):
         details_nodes = node.xpath(
             './ns:NtryDtls/ns:TxDtls', namespaces={'ns': ns})
         if len(details_nodes) == 0:
+            transaction['data'] = etree.tostring(node)
             yield transaction
             return
         transaction_base = transaction

account_bank_statement_import_mt940_nl_ing/mt940.py

@@ -1,23 +1,7 @@
 # -*- coding: utf-8 -*-
 """Implement BankStatementParser for MT940 IBAN ING files."""
-##############################################################################
-#
-#    Copyright (C) 2014-2015 Therp BV <http://therp.nl>.
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as published
-#    by the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
+# © 2014-2016 Therp BV <http://therp.nl>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 import re
 from openerp.addons.account_bank_statement_import_mt940_base.mt940 import (
     MT940, str2amount, get_subfields, handle_common_subfields)
@@ -48,7 +32,7 @@ class MT940Parser(MT940):
         self.current_transaction.transferred_amount = (
             str2amount(parsed_data['sign'], parsed_data['amount']))
         self.current_transaction.eref = parsed_data['reference']
-        self.current_transaction.id = parsed_data['ingid']
+        self.current_transaction.transaction_id = parsed_data['ingid']
 
     def handle_tag_86(self, data):
         """Parse 86 tag containing reference data."""