[11.0] [MIG] account_bank_statement_import_mt940_base.

Update version to 11.

Get balance start even if currency code is already set.

Add SNS test, add header_lines parameter in parse file.

Update code, add pre process datafile to change from different format to only one, statements as dict starting with {4:..}, split file to import multiple statements at once, the check of date of statement import not implemented, be carefully not to import twice the same statements.

Update flake.

Add wrong file.

account_bank_statement_import_mt940_base/README.rst

@@ -1,5 +1,5 @@
-.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
-   :target: http://www.gnu.org/licenses/agpl
+.. image:: https://img.shields.io/badge/license-AGPL--3-blue.png
+   :target: https://www.gnu.org/licenses/agpl
    :alt: License: AGPL-3
 
 ====================
@@ -36,6 +36,7 @@ Contributors
 * Stefan Rijnhart <srijnhart@therp.nl>
 * Ronald Portier <rportier@therp.nl>
 * Andrea Stirpe <a.stirpe@onestein.nl>
+* Fekete Mihai <mihai.fekete@forbiom.eu>
 
 Maintainer
 ----------

account_bank_statement_import_mt940_base/__manifest__.py

@@ -1,25 +1,8 @@
-# -*- coding: utf-8 -*-
-##############################################################################
-#
-#    Copyright (C) 2013-2015 Therp BV <http://therp.nl>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as published
-#    by the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
+# Copyright (C) 2013-2015 Therp BV <http://therp.nl>
+
 {
     'name': 'MT940 Bank Statements Import',
-    'version': '10.0.1.0.0',
+    'version': '11.0.1.0.0',
     'license': 'AGPL-3',
     'author': 'Odoo Community Association (OCA), Therp BV',
     'website': 'https://github.com/OCA/bank-statement-import',

account_bank_statement_import_mt940_base/mt940.py

@@ -1,23 +1,7 @@
-# -*- coding: utf-8 -*-
+# Copyright (C) 2014-2015 Therp BV <http://therp.nl>.
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 """Generic parser for MT940 files, base for customized versions per bank."""
-##############################################################################
-#
-#    Copyright (C) 2014-2015 Therp BV <http://therp.nl>.
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as
-#    published by the Free Software Foundation, either version 3 of the
-#    License, or (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
+
 import re
 import logging
 from datetime import datetime
@@ -138,35 +122,65 @@ class MT940(object):
                 (line[:12], self.mt940_type)
             )
 
-    def parse(self, data):
+    def is_mt940_statement(self, line):
+        """determine if line is the start of a statement"""
+        if not bool(line.startswith('{4:')):
+            raise ValueError(
+                'The pre processed match %s does not seem to be a'
+                ' valid %s MT940 format bank statement. Every statement'
+                ' should start be a dict starting with {4:..' % line
+            )
+
+    def pre_process_data(self, data):
+        matches = []
+        self.is_mt940(line=data)
+        data = data.replace(
+            '-}', '}').replace('}{', '}\r\n{').replace('\r\n', '\n')
+        if data.startswith(':940:'):
+            for statement in data.replace(':940:', '').split(':20:'):
+                match = '{4:\n:20:' + statement + '}'
+                matches.append(match)
+        else:
+            tag_re = re.compile(
+                r'(\{4:[^{}]+\})',
+                re.MULTILINE)
+            matches = tag_re.findall(data)
+        return matches
+
+    def parse(self, data, header_lines=None):
         """Parse mt940 bank statement file contents."""
-        self.is_mt940(data)
-        iterator = data.replace('\r\n', '\n').split('\n').__iter__()
-        line = None
-        record_line = ''
-        try:
-            while True:
-                if not self.current_statement:
-                    self.handle_header(line, iterator)
-                line = iterator.next()
-                if not self.is_tag(line) and not self.is_footer(line):
-                    record_line = self.add_record_line(line, record_line)
-                    continue
+        data = data.decode()
+        matches = self.pre_process_data(data)
+        for match in matches:
+            self.is_mt940_statement(line=match)
+            iterator = '\n'.join(
+                match.split('\n')[1:-1]).split('\n').__iter__()
+            line = None
+            record_line = ''
+            try:
+                while True:
+                    if not self.current_statement:
+                        self.handle_header(line, iterator,
+                                           header_lines=header_lines)
+                    line = next(iterator)
+                    if not self.is_tag(line) and not self.is_footer(line):
+                        record_line = self.add_record_line(line, record_line)
+                        continue
+                    if record_line:
+                        self.handle_record(record_line)
+                    if self.is_footer(line):
+                        self.handle_footer(line, iterator)
+                        record_line = ''
+                        continue
+                    record_line = line
+            except StopIteration:
+                pass
+            if self.current_statement:
                 if record_line:
                     self.handle_record(record_line)
-                if self.is_footer(line):
-                    self.handle_footer(line, iterator)
                     record_line = ''
-                    continue
-                record_line = line
-        except StopIteration:
-            pass
-        if self.current_statement:
-            if record_line:
-                self.handle_record(record_line)
-                record_line = ''
-            self.statements.append(self.current_statement)
-            self.current_statement = None
+                self.statements.append(self.current_statement)
+                self.current_statement = None
         return self.currency_code, self.account_number, self.statements
 
     def add_record_line(self, line, record_line):
@@ -181,10 +195,12 @@ class MT940(object):
         """determine if a line has a tag"""
         return line and bool(re.match(self.tag_regex, line))
 
-    def handle_header(self, dummy_line, iterator):
+    def handle_header(self, dummy_line, iterator, header_lines=None):
         """skip header lines, create current statement"""
-        for dummy_i in range(self.header_lines):
-            iterator.next()
+        if not header_lines:
+            header_lines = self.header_lines
+        for dummy_i in range(header_lines):
+            next(iterator)
         self.current_statement = {
             'name': None,
             'date': None,
@@ -229,10 +245,13 @@ class MT940(object):
         # statement for each 20: tag encountered.
         if not self.currency_code:
             self.currency_code = data[7:10]
-            self.current_statement['balance_start'] = str2amount(
-                data[0],
-                data[10:]
-            )
+        self.current_statement['balance_start'] = str2amount(
+            data[0],
+            data[10:]
+        )
+        if not self.current_statement['date']:
+            self.current_statement['date'] = datetime.strptime(data[1:7],
+                                                               '%y%m%d')
 
     def handle_tag_61(self, data):
         """get transaction values"""
@@ -271,7 +290,7 @@ class MT940(object):
         statement_name = self.current_statement['name'] or ''
         test_empty_id = re.sub(r'[\s0]', '', statement_name)
         is_account_number = statement_name.startswith(self.account_number)
-        if ((not test_empty_id) or is_account_number):
+        if not test_empty_id or is_account_number:
             self.current_statement['name'] = '%s-%s' % (
                 self.account_number,
                 self.current_statement['date'].strftime('%Y-%m-%d'),

account_bank_statement_import_mt940_base/test_files/test-sns.940

@@ -0,0 +1,78 @@
+{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
+:20:0000000000
+:25:NL05SNSB0908244436
+:28C:361/1
+:60F:C171227EUR3026,96
+:61:1712271227D713,13NOVBNL49RABO0166416932
+gerrits glas en schilderwerk
+:86:NL49RABO0166416932 gerrits glas en schilderwerk
+                                                                 
+Factuur 17227/248/20                                             
+                                                                 
+                                                                 
+                                                                 
+:61:1712271227D10,18NBEA//228ohu/972795
+:86: 
+                                                                 
+Jumbo Wijchen B.V.    >WIJCHEN  27.12.2017 13U38 KV005 4XZJ4Z   M
+CC:5411 Contactloze betaling                                     
+                                                                 
+                                                                 
+:61:1712271227D13,52NINCNL94INGB0000869000
+vitens nv
+:86:NL94INGB0000869000 vitens nv
+                                                                 
+Europese incasso door:VITENS NV NL-Factuurnr 072304597540 VNKlant
+nr 00000000000 BTW 0,77PC 6605 DW 6223, DEC-Incassant ID: NL84ZZZ0
+50695810000-Kenmerk Machtiging: bla      
+                                                                 
+:61:1712271227D25,61NBEA//229hro/195867
+:86: 
+                                                                 
+Albert Heijn 1370     >WIJCHEN  27.12.2017 19U13 KV006 70X708   M
+CC:5411                                                          
+                                                                 
+                                                                 
+:62F:C171227EUR2264,52
+-}{5:}
+{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
+:20:0000000000
+:25:NL05SNSB0908244436
+:28C:362/1
+:60F:C171228EUR2264,52
+:61:1712281228D10,95NINCNL40RABO0127859497
+antagonist b.v.
+:86:NL40RABO0127859497 antagonist b.v.
+                                                                 
+Europese incasso door:ANTAGONIST B.V. NL-DDWPN954156 AI529942Cweb
+share.nl-Incassant ID: NL65ZZZ091364410000-Kenmerk Machtiging: bla                                        
+                                                                 
+:61:1712281228C0,00NDIV
+:86: 
+                                                                 
+Mobiel betalen is vanaf nu      niet meer mogelijk voor         B
+ERG M A VAN DEN                met Lenovo P2                     
+                                                                 
+                                                                 
+:62F:C171228EUR2253,57
+-}{5:}
+{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
+:20:0000000000
+:25:NL05SNSB0908244436
+:28C:363/1
+:60F:C171229EUR2253,57
+:61:1712291229D907,29NINCNL19ABNA0427093546
+florius
+:86:NL19ABNA0427093546 florius
+                                                                 
+Europese incasso door:FLORIUS NL-Verschuldigde bedragen PERIODE 1
+2-2017-Incassant ID: NL42ZZZ080242850000-Kenmerk Machtiging: bla                                
+                                                                 
+:61:1712291229D35,00NINCNL28DEUT0265186439
+stichting derdengelden bucka
+:86:NL28DEUT0265186439 stichting derdengelden bucka
+                                                                 
+Europese incasso door:STICHTING DERDENGELDEN BUCKAROO-T-Mobile Th
+uis B.V.: Rekening dec 2017. Bekijk je rekening op thuismy.t-mobi
+le.nl-Incassant ID: NL39ZZZ302317620000-Kenmerk Machtiging: bla
+-}{5:}

account_bank_statement_import_mt940_base/test_files/test-wrong-file.940

@@ -0,0 +1,11 @@
+:9401:
+:20:940S140102
+:25:NL34RABO0142623393 EUR
+:28C:0
+:60F:C131231EUR000000004433,52
+:61:140102C000000000400,00N541NONREF
+NL66RABO0160878799
+:86:/ORDP//NAME/R. SMITH/ADDR/Green market 74 3311BE Sheepcity Nederl
+and NL/REMI/Test money paid by other partner:
+/ISDT/2014-01-02
+:62F:C140102EUR000000004833,52

account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Onestein (<http://www.onestein.eu>)
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 
@@ -48,13 +47,38 @@ class TestImport(TransactionCase):
             'bank_account_id': bank2.id,
             'currency_id': self.env.ref('base.EUR').id,
         })
+        bank3 = self.env['res.partner.bank'].create({
+            'acc_number': 'NL05SNSB0908244436',
+            'partner_id': self.env.ref('base.main_partner').id,
+            'company_id': self.env.ref('base.main_company').id,
+            'bank_id': self.env.ref('base.res_bank_1').id,
+        })
+        self.env['account.journal'].create({
+            'name': 'Bank Journal - (test3 mt940)',
+            'code': 'TBNK3MT940',
+            'type': 'bank',
+            'bank_account_id': bank3.id,
+            'currency_id': self.env.ref('base.EUR').id,
+        })
 
         self.data =\
             "/BENM//NAME/Cost/REMI/Period 01-10-2013 t/m 31-12-2013/ISDT/20"
         self.codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', 'ISDT', 'REMI']
 
+    def test_wrong_file_import(self):
+        """Test wrong file import."""
+        testfile = get_module_resource(
+            'account_bank_statement_import_mt940_base',
+            'test_files',
+            'test-wrong-file.940',
+        )
+        parser = MT940()
+        datafile = open(testfile, 'rb').read()
+        with self.assertRaises(ValueError):
+            parser.parse(datafile, header_lines=1)
+
     def test_statement_import(self):
-        """Test correct creation of single statement."""
+        """Test correct creation of single statement ING."""
 
         def _prepare_statement_lines(statements):
             transact = self.transactions[0]
@@ -72,7 +96,7 @@ class TestImport(TransactionCase):
         )
         parser = MT940()
         datafile = open(testfile, 'rb').read()
-        statements = parser.parse(datafile)
+        statements = parser.parse(datafile, header_lines=1)
 
         _prepare_statement_lines(statements)
 
@@ -89,8 +113,7 @@ class TestImport(TransactionCase):
 
             transact = self.transactions[0]
             for statement in self.env['account.bank.statement'].browse(
-                action['context']['statement_ids']
-            ):
+                    action['context']['statement_ids']):
                 for line in statement.line_ids:
                     self.assertTrue(
                         line.bank_account_id.acc_number ==
@@ -121,7 +144,7 @@ class TestImport(TransactionCase):
         handle_common_subfields(transaction, subfields)
 
     def test_statement_import2(self):
-        """Test correct creation of single statement."""
+        """Test correct creation of single statement RABO."""
 
         def _prepare_statement_lines(statements):
             transact = self.transactions[0]
@@ -141,7 +164,7 @@ class TestImport(TransactionCase):
         parser.header_regex = '^:940:'  # Start of header
         parser.header_lines = 1  # Number of lines to skip
         datafile = open(testfile, 'rb').read()
-        statements = parser.parse(datafile)
+        statements = parser.parse(datafile, header_lines=1)
 
         _prepare_statement_lines(statements)
 
@@ -155,11 +178,11 @@ class TestImport(TransactionCase):
             action = self.env['account.bank.statement.import'].create({
                 'data_file': base64.b64encode(datafile),
             }).import_file()
-
+            # The file contains 4 statements, but only 2 with transactions
+            self.assertTrue(len(action['context']['statement_ids']) == 2)
             transact = self.transactions[0]
             for statement in self.env['account.bank.statement'].browse(
-                action['context']['statement_ids']
-            ):
+                    action['context']['statement_ids']):
                 for line in statement.line_ids:
                     self.assertTrue(
                         line.bank_account_id.acc_number ==
@@ -168,3 +191,49 @@ class TestImport(TransactionCase):
                     self.assertTrue(line.date)
                     self.assertTrue(line.name == transact['name'])
                     self.assertTrue(line.ref == transact['ref'])
+
+    def test_statement_import3(self):
+        """Test correct creation of multiple statements SNS."""
+
+        def _prepare_statement_lines(statements):
+            transact = self.transactions[0]
+            for st_vals in statements[2]:
+                for line_vals in st_vals['transactions']:
+                    line_vals['amount'] = transact['amount']
+                    line_vals['name'] = transact['name']
+                    line_vals['account_number'] = transact['account_number']
+                    line_vals['ref'] = transact['ref']
+
+        testfile = get_module_resource(
+            'account_bank_statement_import_mt940_base',
+            'test_files',
+            'test-sns.940',
+        )
+        parser = MT940()
+        datafile = open(testfile, 'rb').read()
+        statements = parser.parse(datafile, header_lines=1)
+
+        _prepare_statement_lines(statements)
+
+        path_addon = 'odoo.addons.account_bank_statement_import.'
+        path_file = 'account_bank_statement_import.'
+        path_class = 'AccountBankStatementImport.'
+        method = path_addon + path_file + path_class + '_parse_file'
+        with patch(method) as my_mock:
+            my_mock.return_value = statements
+
+            action = self.env['account.bank.statement.import'].create({
+                'data_file': base64.b64encode(datafile),
+            }).import_file()
+            self.assertTrue(len(action['context']['statement_ids']) == 3)
+            transact = self.transactions[-1]
+            for statement in self.env['account.bank.statement'].browse(
+                    action['context']['statement_ids'][-1]):
+                for line in statement.line_ids:
+                    self.assertTrue(
+                        line.bank_account_id.acc_number ==
+                        transact['account_number'])
+                    self.assertTrue(line.amount == transact['amount'])
+                    self.assertTrue(line.date == statement.date)
+                    self.assertTrue(line.name == transact['name'])
+                    self.assertTrue(line.ref == transact['ref'])