From fe73418dffdbd6d5cb94fb8312d3659093fd0579 Mon Sep 17 00:00:00 2001 From: Fekete Mihai Date: Mon, 11 Dec 2017 01:12:28 -0800 Subject: [PATCH] [11.0] [MIG] account_bank_statement_import_mt940_base. Update version to 11. Get balance start even if currency code is already set. Add SNS test, add header_lines parameter in parse file. Update code, add pre process datafile to change from different format to only one, statements as dict starting with {4:..}, split file to import multiple statements at once, the check of date of statement import not implemented, be carefully not to import twice the same statements. Update flake. Add wrong file. --- .../README.rst | 5 +- .../__manifest__.py | 23 +--- .../mt940.py | 123 ++++++++++-------- .../test_files/test-sns.940 | 78 +++++++++++ .../test_files/test-wrong-file.940 | 11 ++ .../tests/test_import_bank_statement.py | 89 +++++++++++-- 6 files changed, 245 insertions(+), 84 deletions(-) create mode 100755 account_bank_statement_import_mt940_base/test_files/test-sns.940 create mode 100644 account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 diff --git a/account_bank_statement_import_mt940_base/README.rst b/account_bank_statement_import_mt940_base/README.rst index 2571371..86c524f 100644 --- a/account_bank_statement_import_mt940_base/README.rst +++ b/account_bank_statement_import_mt940_base/README.rst @@ -1,5 +1,5 @@ -.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg - :target: http://www.gnu.org/licenses/agpl +.. image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: https://www.gnu.org/licenses/agpl :alt: License: AGPL-3 ==================== @@ -36,6 +36,7 @@ Contributors * Stefan Rijnhart * Ronald Portier * Andrea Stirpe +* Fekete Mihai Maintainer ---------- diff --git a/account_bank_statement_import_mt940_base/__manifest__.py b/account_bank_statement_import_mt940_base/__manifest__.py index 212734b..a88409e 100644 --- a/account_bank_statement_import_mt940_base/__manifest__.py +++ b/account_bank_statement_import_mt940_base/__manifest__.py @@ -1,25 +1,8 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# Copyright (C) 2013-2015 Therp BV -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## +# Copyright (C) 2013-2015 Therp BV + { 'name': 'MT940 Bank Statements Import', - 'version': '10.0.1.0.0', + 'version': '11.0.1.0.0', 'license': 'AGPL-3', 'author': 'Odoo Community Association (OCA), Therp BV', 'website': 'https://github.com/OCA/bank-statement-import', diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 24a6969..160e8a8 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -1,23 +1,7 @@ -# -*- coding: utf-8 -*- +# Copyright (C) 2014-2015 Therp BV . +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). """Generic parser for MT940 files, base for customized versions per bank.""" -############################################################################## -# -# Copyright (C) 2014-2015 Therp BV . -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## + import re import logging from datetime import datetime @@ -138,35 +122,65 @@ class MT940(object): (line[:12], self.mt940_type) ) - def parse(self, data): + def is_mt940_statement(self, line): + """determine if line is the start of a statement""" + if not bool(line.startswith('{4:')): + raise ValueError( + 'The pre processed match %s does not seem to be a' + ' valid %s MT940 format bank statement. Every statement' + ' should start be a dict starting with {4:..' % line + ) + + def pre_process_data(self, data): + matches = [] + self.is_mt940(line=data) + data = data.replace( + '-}', '}').replace('}{', '}\r\n{').replace('\r\n', '\n') + if data.startswith(':940:'): + for statement in data.replace(':940:', '').split(':20:'): + match = '{4:\n:20:' + statement + '}' + matches.append(match) + else: + tag_re = re.compile( + r'(\{4:[^{}]+\})', + re.MULTILINE) + matches = tag_re.findall(data) + return matches + + def parse(self, data, header_lines=None): """Parse mt940 bank statement file contents.""" - self.is_mt940(data) - iterator = data.replace('\r\n', '\n').split('\n').__iter__() - line = None - record_line = '' - try: - while True: - if not self.current_statement: - self.handle_header(line, iterator) - line = iterator.next() - if not self.is_tag(line) and not self.is_footer(line): - record_line = self.add_record_line(line, record_line) - continue + data = data.decode() + matches = self.pre_process_data(data) + for match in matches: + self.is_mt940_statement(line=match) + iterator = '\n'.join( + match.split('\n')[1:-1]).split('\n').__iter__() + line = None + record_line = '' + try: + while True: + if not self.current_statement: + self.handle_header(line, iterator, + header_lines=header_lines) + line = next(iterator) + if not self.is_tag(line) and not self.is_footer(line): + record_line = self.add_record_line(line, record_line) + continue + if record_line: + self.handle_record(record_line) + if self.is_footer(line): + self.handle_footer(line, iterator) + record_line = '' + continue + record_line = line + except StopIteration: + pass + if self.current_statement: if record_line: self.handle_record(record_line) - if self.is_footer(line): - self.handle_footer(line, iterator) record_line = '' - continue - record_line = line - except StopIteration: - pass - if self.current_statement: - if record_line: - self.handle_record(record_line) - record_line = '' - self.statements.append(self.current_statement) - self.current_statement = None + self.statements.append(self.current_statement) + self.current_statement = None return self.currency_code, self.account_number, self.statements def add_record_line(self, line, record_line): @@ -181,10 +195,12 @@ class MT940(object): """determine if a line has a tag""" return line and bool(re.match(self.tag_regex, line)) - def handle_header(self, dummy_line, iterator): + def handle_header(self, dummy_line, iterator, header_lines=None): """skip header lines, create current statement""" - for dummy_i in range(self.header_lines): - iterator.next() + if not header_lines: + header_lines = self.header_lines + for dummy_i in range(header_lines): + next(iterator) self.current_statement = { 'name': None, 'date': None, @@ -229,10 +245,13 @@ class MT940(object): # statement for each 20: tag encountered. if not self.currency_code: self.currency_code = data[7:10] - self.current_statement['balance_start'] = str2amount( - data[0], - data[10:] - ) + self.current_statement['balance_start'] = str2amount( + data[0], + data[10:] + ) + if not self.current_statement['date']: + self.current_statement['date'] = datetime.strptime(data[1:7], + '%y%m%d') def handle_tag_61(self, data): """get transaction values""" @@ -271,7 +290,7 @@ class MT940(object): statement_name = self.current_statement['name'] or '' test_empty_id = re.sub(r'[\s0]', '', statement_name) is_account_number = statement_name.startswith(self.account_number) - if ((not test_empty_id) or is_account_number): + if not test_empty_id or is_account_number: self.current_statement['name'] = '%s-%s' % ( self.account_number, self.current_statement['date'].strftime('%Y-%m-%d'), diff --git a/account_bank_statement_import_mt940_base/test_files/test-sns.940 b/account_bank_statement_import_mt940_base/test_files/test-sns.940 new file mode 100755 index 0000000..a1b032c --- /dev/null +++ b/account_bank_statement_import_mt940_base/test_files/test-sns.940 @@ -0,0 +1,78 @@ +{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4: +:20:0000000000 +:25:NL05SNSB0908244436 +:28C:361/1 +:60F:C171227EUR3026,96 +:61:1712271227D713,13NOVBNL49RABO0166416932 +gerrits glas en schilderwerk +:86:NL49RABO0166416932 gerrits glas en schilderwerk + +Factuur 17227/248/20 + + + +:61:1712271227D10,18NBEA//228ohu/972795 +:86: + +Jumbo Wijchen B.V. >WIJCHEN 27.12.2017 13U38 KV005 4XZJ4Z M +CC:5411 Contactloze betaling + + +:61:1712271227D13,52NINCNL94INGB0000869000 +vitens nv +:86:NL94INGB0000869000 vitens nv + +Europese incasso door:VITENS NV NL-Factuurnr 072304597540 VNKlant +nr 00000000000 BTW 0,77PC 6605 DW 6223, DEC-Incassant ID: NL84ZZZ0 +50695810000-Kenmerk Machtiging: bla + +:61:1712271227D25,61NBEA//229hro/195867 +:86: + +Albert Heijn 1370 >WIJCHEN 27.12.2017 19U13 KV006 70X708 M +CC:5411 + + +:62F:C171227EUR2264,52 +-}{5:} +{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4: +:20:0000000000 +:25:NL05SNSB0908244436 +:28C:362/1 +:60F:C171228EUR2264,52 +:61:1712281228D10,95NINCNL40RABO0127859497 +antagonist b.v. +:86:NL40RABO0127859497 antagonist b.v. + +Europese incasso door:ANTAGONIST B.V. NL-DDWPN954156 AI529942Cweb +share.nl-Incassant ID: NL65ZZZ091364410000-Kenmerk Machtiging: bla + +:61:1712281228C0,00NDIV +:86: + +Mobiel betalen is vanaf nu niet meer mogelijk voor B +ERG M A VAN DEN met Lenovo P2 + + +:62F:C171228EUR2253,57 +-}{5:} +{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4: +:20:0000000000 +:25:NL05SNSB0908244436 +:28C:363/1 +:60F:C171229EUR2253,57 +:61:1712291229D907,29NINCNL19ABNA0427093546 +florius +:86:NL19ABNA0427093546 florius + +Europese incasso door:FLORIUS NL-Verschuldigde bedragen PERIODE 1 +2-2017-Incassant ID: NL42ZZZ080242850000-Kenmerk Machtiging: bla + +:61:1712291229D35,00NINCNL28DEUT0265186439 +stichting derdengelden bucka +:86:NL28DEUT0265186439 stichting derdengelden bucka + +Europese incasso door:STICHTING DERDENGELDEN BUCKAROO-T-Mobile Th +uis B.V.: Rekening dec 2017. Bekijk je rekening op thuismy.t-mobi +le.nl-Incassant ID: NL39ZZZ302317620000-Kenmerk Machtiging: bla +-}{5:} diff --git a/account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 b/account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 new file mode 100644 index 0000000..ae75738 --- /dev/null +++ b/account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 @@ -0,0 +1,11 @@ +:9401: +:20:940S140102 +:25:NL34RABO0142623393 EUR +:28C:0 +:60F:C131231EUR000000004433,52 +:61:140102C000000000400,00N541NONREF +NL66RABO0160878799 +:86:/ORDP//NAME/R. SMITH/ADDR/Green market 74 3311BE Sheepcity Nederl +and NL/REMI/Test money paid by other partner: +/ISDT/2014-01-02 +:62F:C140102EUR000000004833,52 diff --git a/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py b/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py index b25ffc9..9ff19e0 100644 --- a/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py +++ b/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2017 Onestein () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). @@ -48,13 +47,38 @@ class TestImport(TransactionCase): 'bank_account_id': bank2.id, 'currency_id': self.env.ref('base.EUR').id, }) + bank3 = self.env['res.partner.bank'].create({ + 'acc_number': 'NL05SNSB0908244436', + 'partner_id': self.env.ref('base.main_partner').id, + 'company_id': self.env.ref('base.main_company').id, + 'bank_id': self.env.ref('base.res_bank_1').id, + }) + self.env['account.journal'].create({ + 'name': 'Bank Journal - (test3 mt940)', + 'code': 'TBNK3MT940', + 'type': 'bank', + 'bank_account_id': bank3.id, + 'currency_id': self.env.ref('base.EUR').id, + }) self.data =\ "/BENM//NAME/Cost/REMI/Period 01-10-2013 t/m 31-12-2013/ISDT/20" self.codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', 'ISDT', 'REMI'] + def test_wrong_file_import(self): + """Test wrong file import.""" + testfile = get_module_resource( + 'account_bank_statement_import_mt940_base', + 'test_files', + 'test-wrong-file.940', + ) + parser = MT940() + datafile = open(testfile, 'rb').read() + with self.assertRaises(ValueError): + parser.parse(datafile, header_lines=1) + def test_statement_import(self): - """Test correct creation of single statement.""" + """Test correct creation of single statement ING.""" def _prepare_statement_lines(statements): transact = self.transactions[0] @@ -72,7 +96,7 @@ class TestImport(TransactionCase): ) parser = MT940() datafile = open(testfile, 'rb').read() - statements = parser.parse(datafile) + statements = parser.parse(datafile, header_lines=1) _prepare_statement_lines(statements) @@ -89,8 +113,7 @@ class TestImport(TransactionCase): transact = self.transactions[0] for statement in self.env['account.bank.statement'].browse( - action['context']['statement_ids'] - ): + action['context']['statement_ids']): for line in statement.line_ids: self.assertTrue( line.bank_account_id.acc_number == @@ -121,7 +144,7 @@ class TestImport(TransactionCase): handle_common_subfields(transaction, subfields) def test_statement_import2(self): - """Test correct creation of single statement.""" + """Test correct creation of single statement RABO.""" def _prepare_statement_lines(statements): transact = self.transactions[0] @@ -141,7 +164,7 @@ class TestImport(TransactionCase): parser.header_regex = '^:940:' # Start of header parser.header_lines = 1 # Number of lines to skip datafile = open(testfile, 'rb').read() - statements = parser.parse(datafile) + statements = parser.parse(datafile, header_lines=1) _prepare_statement_lines(statements) @@ -155,11 +178,11 @@ class TestImport(TransactionCase): action = self.env['account.bank.statement.import'].create({ 'data_file': base64.b64encode(datafile), }).import_file() - + # The file contains 4 statements, but only 2 with transactions + self.assertTrue(len(action['context']['statement_ids']) == 2) transact = self.transactions[0] for statement in self.env['account.bank.statement'].browse( - action['context']['statement_ids'] - ): + action['context']['statement_ids']): for line in statement.line_ids: self.assertTrue( line.bank_account_id.acc_number == @@ -168,3 +191,49 @@ class TestImport(TransactionCase): self.assertTrue(line.date) self.assertTrue(line.name == transact['name']) self.assertTrue(line.ref == transact['ref']) + + def test_statement_import3(self): + """Test correct creation of multiple statements SNS.""" + + def _prepare_statement_lines(statements): + transact = self.transactions[0] + for st_vals in statements[2]: + for line_vals in st_vals['transactions']: + line_vals['amount'] = transact['amount'] + line_vals['name'] = transact['name'] + line_vals['account_number'] = transact['account_number'] + line_vals['ref'] = transact['ref'] + + testfile = get_module_resource( + 'account_bank_statement_import_mt940_base', + 'test_files', + 'test-sns.940', + ) + parser = MT940() + datafile = open(testfile, 'rb').read() + statements = parser.parse(datafile, header_lines=1) + + _prepare_statement_lines(statements) + + path_addon = 'odoo.addons.account_bank_statement_import.' + path_file = 'account_bank_statement_import.' + path_class = 'AccountBankStatementImport.' + method = path_addon + path_file + path_class + '_parse_file' + with patch(method) as my_mock: + my_mock.return_value = statements + + action = self.env['account.bank.statement.import'].create({ + 'data_file': base64.b64encode(datafile), + }).import_file() + self.assertTrue(len(action['context']['statement_ids']) == 3) + transact = self.transactions[-1] + for statement in self.env['account.bank.statement'].browse( + action['context']['statement_ids'][-1]): + for line in statement.line_ids: + self.assertTrue( + line.bank_account_id.acc_number == + transact['account_number']) + self.assertTrue(line.amount == transact['amount']) + self.assertTrue(line.date == statement.date) + self.assertTrue(line.name == transact['name']) + self.assertTrue(line.ref == transact['ref'])