Backport of V11 developments (from #137)

account_bank_statement_import_mt940_base/README.rst

@@ -1,5 +1,5 @@
-.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
-   :target: http://www.gnu.org/licenses/agpl
+.. image:: https://img.shields.io/badge/license-AGPL--3-blue.png
+   :target: https://www.gnu.org/licenses/agpl
    :alt: License: AGPL-3
 
 ====================
@@ -36,6 +36,7 @@ Contributors
 * Stefan Rijnhart <srijnhart@therp.nl>
 * Ronald Portier <rportier@therp.nl>
 * Andrea Stirpe <a.stirpe@onestein.nl>
+* Fekete Mihai <mihai.fekete@forbiom.eu>
 
 Maintainer
 ----------

account_bank_statement_import_mt940_base/__manifest__.py

@@ -1,22 +1,6 @@
 # -*- coding: utf-8 -*-
-##############################################################################
-#
 # Copyright (C) 2013-2015 Therp BV <http://therp.nl>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as published
-#    by the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
+
 {
     'name': 'MT940 Bank Statements Import',
     'version': '10.0.1.0.0',

account_bank_statement_import_mt940_base/mt940.py

@@ -1,23 +1,7 @@
-# -*- coding: utf-8 -*-
-"""Generic parser for MT940 files, base for customized versions per bank."""
-##############################################################################
-#
 # Copyright (C) 2014-2015 Therp BV <http://therp.nl>.
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as
-#    published by the Free Software Foundation, either version 3 of the
-#    License, or (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+"""Generic parser for MT940 files, base for customized versions per bank."""
+
 import re
 import logging
 from datetime import datetime
@@ -138,16 +122,46 @@ class MT940(object):
                 (line[:12], self.mt940_type)
             )
 
-    def parse(self, data):
+    def is_mt940_statement(self, line):
+        """determine if line is the start of a statement"""
+        if not bool(line.startswith('{4:')):
+            raise ValueError(
+                'The pre processed match %s does not seem to be a'
+                ' valid %s MT940 format bank statement. Every statement'
+                ' should start be a dict starting with {4:..' % line
+            )
+
+    def pre_process_data(self, data):
+        matches = []
+        self.is_mt940(line=data)
+        data = data.replace(
+            '-}', '}').replace('}{', '}\r\n{').replace('\r\n', '\n')
+        if data.startswith(':940:'):
+            for statement in data.replace(':940:', '').split(':20:'):
+                match = '{4:\n:20:' + statement + '}'
+                matches.append(match)
+        else:
+            tag_re = re.compile(
+                r'(\{4:[^{}]+\})',
+                re.MULTILINE)
+            matches = tag_re.findall(data)
+        return matches
+
+    def parse(self, data, header_lines=None):
         """Parse mt940 bank statement file contents."""
-        self.is_mt940(data)
-        iterator = data.replace('\r\n', '\n').split('\n').__iter__()
+        data = data.decode()
+        matches = self.pre_process_data(data)
+        for match in matches:
+            self.is_mt940_statement(line=match)
+            iterator = '\n'.join(
+                match.split('\n')[1:-1]).split('\n').__iter__()
             line = None
             record_line = ''
             try:
                 while True:
                     if not self.current_statement:
-                    self.handle_header(line, iterator)
+                        self.handle_header(line, iterator,
+                                           header_lines=header_lines)
                     line = iterator.next()
                     if not self.is_tag(line) and not self.is_footer(line):
                         record_line = self.add_record_line(line, record_line)
@@ -181,9 +195,11 @@ class MT940(object):
         """determine if a line has a tag"""
         return line and bool(re.match(self.tag_regex, line))
 
-    def handle_header(self, dummy_line, iterator):
+    def handle_header(self, dummy_line, iterator, header_lines=None):
         """skip header lines, create current statement"""
-        for dummy_i in range(self.header_lines):
+        if not header_lines:
+            header_lines = self.header_lines
+        for dummy_i in range(header_lines):
             iterator.next()
         self.current_statement = {
             'name': None,
@@ -233,6 +249,9 @@ class MT940(object):
             data[0],
             data[10:]
         )
+        if not self.current_statement['date']:
+            self.current_statement['date'] = datetime.strptime(data[1:7],
+                                                               '%y%m%d')
 
     def handle_tag_61(self, data):
         """get transaction values"""
@@ -271,7 +290,7 @@ class MT940(object):
         statement_name = self.current_statement['name'] or ''
         test_empty_id = re.sub(r'[\s0]', '', statement_name)
         is_account_number = statement_name.startswith(self.account_number)
-        if ((not test_empty_id) or is_account_number):
+        if not test_empty_id or is_account_number:
             self.current_statement['name'] = '%s-%s' % (
                 self.account_number,
                 self.current_statement['date'].strftime('%Y-%m-%d'),

account_bank_statement_import_mt940_base/test_files/test-sns.940

@@ -0,0 +1,78 @@
+{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
+:20:0000000000
+:25:NL05SNSB0908244436
+:28C:361/1
+:60F:C171227EUR3026,96
+:61:1712271227D713,13NOVBNL49RABO0166416932
+gerrits glas en schilderwerk
+:86:NL49RABO0166416932 gerrits glas en schilderwerk
+                                                                 
+Factuur 17227/248/20                                             
+                                                                 
+                                                                 
+                                                                 
+:61:1712271227D10,18NBEA//228ohu/972795
+:86: 
+                                                                 
+Jumbo Wijchen B.V.    >WIJCHEN  27.12.2017 13U38 KV005 4XZJ4Z   M
+CC:5411 Contactloze betaling                                     
+                                                                 
+                                                                 
+:61:1712271227D13,52NINCNL94INGB0000869000
+vitens nv
+:86:NL94INGB0000869000 vitens nv
+                                                                 
+Europese incasso door:VITENS NV NL-Factuurnr 072304597540 VNKlant
+nr 00000000000 BTW 0,77PC 6605 DW 6223, DEC-Incassant ID: NL84ZZZ0
+50695810000-Kenmerk Machtiging: bla      
+                                                                 
+:61:1712271227D25,61NBEA//229hro/195867
+:86: 
+                                                                 
+Albert Heijn 1370     >WIJCHEN  27.12.2017 19U13 KV006 70X708   M
+CC:5411                                                          
+                                                                 
+                                                                 
+:62F:C171227EUR2264,52
+-}{5:}
+{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
+:20:0000000000
+:25:NL05SNSB0908244436
+:28C:362/1
+:60F:C171228EUR2264,52
+:61:1712281228D10,95NINCNL40RABO0127859497
+antagonist b.v.
+:86:NL40RABO0127859497 antagonist b.v.
+                                                                 
+Europese incasso door:ANTAGONIST B.V. NL-DDWPN954156 AI529942Cweb
+share.nl-Incassant ID: NL65ZZZ091364410000-Kenmerk Machtiging: bla                                        
+                                                                 
+:61:1712281228C0,00NDIV
+:86: 
+                                                                 
+Mobiel betalen is vanaf nu      niet meer mogelijk voor         B
+ERG M A VAN DEN                met Lenovo P2                     
+                                                                 
+                                                                 
+:62F:C171228EUR2253,57
+-}{5:}
+{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
+:20:0000000000
+:25:NL05SNSB0908244436
+:28C:363/1
+:60F:C171229EUR2253,57
+:61:1712291229D907,29NINCNL19ABNA0427093546
+florius
+:86:NL19ABNA0427093546 florius
+                                                                 
+Europese incasso door:FLORIUS NL-Verschuldigde bedragen PERIODE 1
+2-2017-Incassant ID: NL42ZZZ080242850000-Kenmerk Machtiging: bla                                
+                                                                 
+:61:1712291229D35,00NINCNL28DEUT0265186439
+stichting derdengelden bucka
+:86:NL28DEUT0265186439 stichting derdengelden bucka
+                                                                 
+Europese incasso door:STICHTING DERDENGELDEN BUCKAROO-T-Mobile Th
+uis B.V.: Rekening dec 2017. Bekijk je rekening op thuismy.t-mobi
+le.nl-Incassant ID: NL39ZZZ302317620000-Kenmerk Machtiging: bla
+-}{5:}

account_bank_statement_import_mt940_base/test_files/test-wrong-file.940

@@ -0,0 +1,11 @@
+:9401:
+:20:940S140102
+:25:NL34RABO0142623393 EUR
+:28C:0
+:60F:C131231EUR000000004433,52
+:61:140102C000000000400,00N541NONREF
+NL66RABO0160878799
+:86:/ORDP//NAME/R. SMITH/ADDR/Green market 74 3311BE Sheepcity Nederl
+and NL/REMI/Test money paid by other partner:
+/ISDT/2014-01-02
+:62F:C140102EUR000000004833,52

account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py

@@ -48,13 +48,38 @@ class TestImport(TransactionCase):
             'bank_account_id': bank2.id,
             'currency_id': self.env.ref('base.EUR').id,
         })
+        bank3 = self.env['res.partner.bank'].create({
+            'acc_number': 'NL05SNSB0908244436',
+            'partner_id': self.env.ref('base.main_partner').id,
+            'company_id': self.env.ref('base.main_company').id,
+            'bank_id': self.env.ref('base.res_bank_1').id,
+        })
+        self.env['account.journal'].create({
+            'name': 'Bank Journal - (test3 mt940)',
+            'code': 'TBNK3MT940',
+            'type': 'bank',
+            'bank_account_id': bank3.id,
+            'currency_id': self.env.ref('base.EUR').id,
+        })
 
         self.data =\
             "/BENM//NAME/Cost/REMI/Period 01-10-2013 t/m 31-12-2013/ISDT/20"
         self.codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', 'ISDT', 'REMI']
 
+    def test_wrong_file_import(self):
+        """Test wrong file import."""
+        testfile = get_module_resource(
+            'account_bank_statement_import_mt940_base',
+            'test_files',
+            'test-wrong-file.940',
+        )
+        parser = MT940()
+        datafile = open(testfile, 'rb').read()
+        with self.assertRaises(ValueError):
+            parser.parse(datafile, header_lines=1)
+
     def test_statement_import(self):
-        """Test correct creation of single statement."""
+        """Test correct creation of single statement ING."""
 
         def _prepare_statement_lines(statements):
             transact = self.transactions[0]
@@ -72,7 +97,7 @@ class TestImport(TransactionCase):
         )
         parser = MT940()
         datafile = open(testfile, 'rb').read()
-        statements = parser.parse(datafile)
+        statements = parser.parse(datafile, header_lines=1)
 
         _prepare_statement_lines(statements)
 
@@ -89,8 +114,7 @@ class TestImport(TransactionCase):
 
             transact = self.transactions[0]
             for statement in self.env['account.bank.statement'].browse(
-                action['context']['statement_ids']
-            ):
+                    action['context']['statement_ids']):
                 for line in statement.line_ids:
                     self.assertTrue(
                         line.bank_account_id.acc_number ==
@@ -121,7 +145,7 @@ class TestImport(TransactionCase):
         handle_common_subfields(transaction, subfields)
 
     def test_statement_import2(self):
-        """Test correct creation of single statement."""
+        """Test correct creation of single statement RABO."""
 
         def _prepare_statement_lines(statements):
             transact = self.transactions[0]
@@ -141,7 +165,7 @@ class TestImport(TransactionCase):
         parser.header_regex = '^:940:'  # Start of header
         parser.header_lines = 1  # Number of lines to skip
         datafile = open(testfile, 'rb').read()
-        statements = parser.parse(datafile)
+        statements = parser.parse(datafile, header_lines=1)
 
         _prepare_statement_lines(statements)
 
@@ -155,11 +179,11 @@ class TestImport(TransactionCase):
             action = self.env['account.bank.statement.import'].create({
                 'data_file': base64.b64encode(datafile),
             }).import_file()
-
+            # The file contains 4 statements, but only 2 with transactions
+            self.assertTrue(len(action['context']['statement_ids']) == 2)
             transact = self.transactions[0]
             for statement in self.env['account.bank.statement'].browse(
-                action['context']['statement_ids']
-            ):
+                    action['context']['statement_ids']):
                 for line in statement.line_ids:
                     self.assertTrue(
                         line.bank_account_id.acc_number ==
@@ -168,3 +192,49 @@ class TestImport(TransactionCase):
                     self.assertTrue(line.date)
                     self.assertTrue(line.name == transact['name'])
                     self.assertTrue(line.ref == transact['ref'])
+
+    def test_statement_import3(self):
+        """Test correct creation of multiple statements SNS."""
+
+        def _prepare_statement_lines(statements):
+            transact = self.transactions[0]
+            for st_vals in statements[2]:
+                for line_vals in st_vals['transactions']:
+                    line_vals['amount'] = transact['amount']
+                    line_vals['name'] = transact['name']
+                    line_vals['account_number'] = transact['account_number']
+                    line_vals['ref'] = transact['ref']
+
+        testfile = get_module_resource(
+            'account_bank_statement_import_mt940_base',
+            'test_files',
+            'test-sns.940',
+        )
+        parser = MT940()
+        datafile = open(testfile, 'rb').read()
+        statements = parser.parse(datafile, header_lines=1)
+
+        _prepare_statement_lines(statements)
+
+        path_addon = 'odoo.addons.account_bank_statement_import.'
+        path_file = 'account_bank_statement_import.'
+        path_class = 'AccountBankStatementImport.'
+        method = path_addon + path_file + path_class + '_parse_file'
+        with patch(method) as my_mock:
+            my_mock.return_value = statements
+
+            action = self.env['account.bank.statement.import'].create({
+                'data_file': base64.b64encode(datafile),
+            }).import_file()
+            self.assertTrue(len(action['context']['statement_ids']) == 3)
+            transact = self.transactions[-1]
+            for statement in self.env['account.bank.statement'].browse(
+                    action['context']['statement_ids'][-1]):
+                for line in statement.line_ids:
+                    self.assertTrue(
+                        line.bank_account_id.acc_number ==
+                        transact['account_number'])
+                    self.assertTrue(line.amount == transact['amount'])
+                    self.assertTrue(line.date == statement.date)
+                    self.assertTrue(line.name == transact['name'])
+                    self.assertTrue(line.ref == transact['ref'])