From 72a6d0b89efab605215b38800b3b9381e7a8d44a Mon Sep 17 00:00:00 2001 From: "Ronald Portier (Therp BV)" Date: Sat, 13 Jun 2015 01:20:18 +0200 Subject: [PATCH 1/6] [ENH] Support zip files. --- account_bank_statement_import_mt940_base/mt940.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 8677bc7..9ddd9ce 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -111,7 +111,7 @@ class MT940(object): This in fact uses the ING syntax, override in others.""" self.mt940_type = 'General' self.header_lines = 3 # Number of lines to skip - self.header_regex = '^0000 01INGBNL2AXXXX|^{1' + self.header_regex = '^0000 01INGBNL2AXXXX|^{1' # Start of header self.footer_regex = '^-}$|^-XXX$' # Stop processing on seeing this self.tag_regex = '^:[0-9]{2}[A-Z]*:' # Start of new tag self.current_statement = None From dc88017b6182387bc3da71e824db234c83fac5b0 Mon Sep 17 00:00:00 2001 From: "Ronald Portier (Therp BV)" Date: Fri, 2 Oct 2015 00:12:32 +0200 Subject: [PATCH 2/6] [FIX] Import exceptions.Warning as UserError. This according to OCA guidelines. Also reformat README.rst to keep lines within limit. --- account_bank_statement_import_mt940_base/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/account_bank_statement_import_mt940_base/__manifest__.py b/account_bank_statement_import_mt940_base/__manifest__.py index 5aee03c..5c672a6 100644 --- a/account_bank_statement_import_mt940_base/__manifest__.py +++ b/account_bank_statement_import_mt940_base/__manifest__.py @@ -19,7 +19,7 @@ ############################################################################## { 'name': 'MT940 Bank Statements Import', - 'version': '8.0.1.1.0', + 'version': '8.0.1.1.1', 'license': 'AGPL-3', 'author': 'Odoo Community Association (OCA), Therp BV', 'website': 'https://github.com/OCA/bank-statement-import', From 8ca3412c5b433eaae99c64f9acb9aac12d1711e8 Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Fri, 14 Oct 2016 13:32:23 +0000 Subject: [PATCH 3/6] [UPD] parse more fields as of (#71) * [UPD] parse more fields for MT940 as of https://www.ing.nl/media/ING_ming_mt940s_24_juli_tcm162-46356.pdf p 8ff * [FIX] There is no CR or LF in mt940 data to be parsed. The calling logic from the MT940 parser concatenates all lines from a mt940 tag. --- .../mt940.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 9ddd9ce..3c36f42 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -83,14 +83,28 @@ def handle_common_subfields(transaction, subfields): for counterpart_field in ['CNTP', 'BENM', 'ORDP']: if counterpart_field in subfields: get_counterpart(transaction, subfields[counterpart_field]) + if not transaction.message: + transaction.message = '' # REMI: Remitter information (text entered by other party on trans.): if 'REMI' in subfields: - transaction.message = ( - '/'.join(x for x in subfields['REMI'] if x)) + transaction.message += ( + subfields['REMI'][2] + # this might look like + # /REMI/USTD/// + # or + # /REMI/STRD/CUR// + if len(subfields['REMI']) >= 3 and subfields['REMI'][0] in [ + 'STRD', 'USTD' + ] + else + '/'.join(x for x in subfields['REMI'] if x) + ) + # EREF: End-to-end reference + if 'EREF' in subfields: + transaction.message += '/'.join(filter(bool, subfields['EREF'])) # Get transaction reference subfield (might vary): if transaction.eref in subfields: - transaction.eref = ''.join( - subfields[transaction.eref]) + transaction.eref = ''.join(subfields[transaction.eref]) class MT940(object): From deb933492cf0ae0f5c2ef15250fb714dee48280b Mon Sep 17 00:00:00 2001 From: Andrea Stirpe Date: Wed, 17 May 2017 10:12:19 +0200 Subject: [PATCH 4/6] Extract method to allow extensions (#109) --- account_bank_statement_import_mt940_base/mt940.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 3c36f42..22dc5a6 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -153,7 +153,7 @@ class MT940(object): self.handle_header(line, iterator) line = iterator.next() if not self.is_tag(line) and not self.is_footer(line): - record_line += line + record_line = self.add_record_line(line, record_line) continue if record_line: self.handle_record(record_line) @@ -172,6 +172,10 @@ class MT940(object): self.current_statement = None return self.statements + def add_record_line(self, line, record_line): + record_line += line + return record_line + def is_footer(self, line): """determine if a line is the footer of a statement""" return line and bool(re.match(self.footer_regex, line)) From f50d296c5401685d650e77d481eb3ebd17fa6dbe Mon Sep 17 00:00:00 2001 From: Andrea Date: Tue, 12 Sep 2017 12:18:22 +0200 Subject: [PATCH 5/6] [10.0][MIG] account_bank_statement_import_mt940_base: port to V10 --- .../README.rst | 21 +-- .../__manifest__.py | 4 +- .../mt940.py | 83 +++++---- .../test_files/test-ing.940 | 61 +++++++ .../test_files/test-rabo.swi | 29 +++ .../tests/__init__.py | 1 + .../tests/test_import_bank_statement.py | 170 ++++++++++++++++++ 7 files changed, 321 insertions(+), 48 deletions(-) create mode 100644 account_bank_statement_import_mt940_base/test_files/test-ing.940 create mode 100644 account_bank_statement_import_mt940_base/test_files/test-rabo.swi create mode 100644 account_bank_statement_import_mt940_base/tests/__init__.py create mode 100644 account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py diff --git a/account_bank_statement_import_mt940_base/README.rst b/account_bank_statement_import_mt940_base/README.rst index 73a280a..2571371 100644 --- a/account_bank_statement_import_mt940_base/README.rst +++ b/account_bank_statement_import_mt940_base/README.rst @@ -1,6 +1,8 @@ .. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg - :alt: License: AGPL-3 + :target: http://www.gnu.org/licenses/agpl + :alt: License: AGPL-3 +==================== Bank Statement MT940 ==================== @@ -10,7 +12,7 @@ interpretation of it, this addon alone won't help you much. It is rather intended to be used by other addons to implement the dialect specific to a certain bank. -See bank_statement_parse_nl_ing_mt940 for an example on how to use it. +See account_bank_statement_import_mt940_nl_ing for an example on how to use it. Known issues / Roadmap ====================== @@ -20,11 +22,10 @@ Known issues / Roadmap Bug Tracker =========== -Bugs are tracked on `GitHub Issues `_. -In case of trouble, please check there if your issue has already been reported. -If you spotted it first, help us smashing it by providing a detailed and welcomed feedback -`here `_. - +Bugs are tracked on `GitHub Issues +`_. In case of trouble, please +check there if your issue has already been reported. If you spotted it first, +help us smash it by providing detailed and welcomed feedback. Credits ======= @@ -34,10 +35,12 @@ Contributors * Stefan Rijnhart * Ronald Portier +* Andrea Stirpe Maintainer ---------- + .. image:: https://odoo-community.org/logo.png :alt: Odoo Community Association :target: https://odoo-community.org @@ -48,6 +51,4 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -To contribute to this module, please visit http://odoo-community.org. -This module should make it easy to migrate bank statement import -modules written for earlies versions of Odoo/OpenERP. +To contribute to this module, please visit https://odoo-community.org. diff --git a/account_bank_statement_import_mt940_base/__manifest__.py b/account_bank_statement_import_mt940_base/__manifest__.py index 5c672a6..212734b 100644 --- a/account_bank_statement_import_mt940_base/__manifest__.py +++ b/account_bank_statement_import_mt940_base/__manifest__.py @@ -19,7 +19,7 @@ ############################################################################## { 'name': 'MT940 Bank Statements Import', - 'version': '8.0.1.1.1', + 'version': '10.0.1.0.0', 'license': 'AGPL-3', 'author': 'Odoo Community Association (OCA), Therp BV', 'website': 'https://github.com/OCA/bank-statement-import', @@ -27,5 +27,5 @@ 'depends': [ 'account_bank_statement_import', ], - 'installable': False + 'installable': True } diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 22dc5a6..24a6969 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -22,9 +22,6 @@ import re import logging from datetime import datetime -from openerp.addons.account_bank_statement_import.parserlib import ( - BankStatement) - def str2amount(sign, amount_str): """Convert sign (C or D) and amount in string to signed amount (float).""" @@ -68,13 +65,11 @@ def get_counterpart(transaction, subfield): if not subfield: return # subfield is empty if len(subfield) >= 1 and subfield[0]: - transaction.remote_account = subfield[0] + transaction.update({'account_number': subfield[0]}) if len(subfield) >= 2 and subfield[1]: - transaction.remote_bank_bic = subfield[1] + transaction.update({'account_bic': subfield[1]}) if len(subfield) >= 3 and subfield[2]: - transaction.remote_owner = subfield[2] - if len(subfield) >= 4 and subfield[3]: - transaction.remote_owner_city = subfield[3] + transaction.update({'partner_name': subfield[2]}) def handle_common_subfields(transaction, subfields): @@ -83,11 +78,11 @@ def handle_common_subfields(transaction, subfields): for counterpart_field in ['CNTP', 'BENM', 'ORDP']: if counterpart_field in subfields: get_counterpart(transaction, subfields[counterpart_field]) - if not transaction.message: - transaction.message = '' + if not transaction.get('name'): + transaction['name'] = '' # REMI: Remitter information (text entered by other party on trans.): if 'REMI' in subfields: - transaction.message += ( + transaction['name'] += ( subfields['REMI'][2] # this might look like # /REMI/USTD/// @@ -101,10 +96,10 @@ def handle_common_subfields(transaction, subfields): ) # EREF: End-to-end reference if 'EREF' in subfields: - transaction.message += '/'.join(filter(bool, subfields['EREF'])) + transaction['name'] += '/'.join(filter(bool, subfields['EREF'])) # Get transaction reference subfield (might vary): - if transaction.eref in subfields: - transaction.eref = ''.join(subfields[transaction.eref]) + if transaction.get('ref') in subfields: + transaction['ref'] = ''.join(subfields[transaction['ref']]) class MT940(object): @@ -131,6 +126,8 @@ class MT940(object): self.current_statement = None self.current_transaction = None self.statements = [] + self.currency_code = None + self.account_number = None def is_mt940(self, line): """determine if a line is the header of a statement""" @@ -170,7 +167,7 @@ class MT940(object): record_line = '' self.statements.append(self.current_statement) self.current_statement = None - return self.statements + return self.currency_code, self.account_number, self.statements def add_record_line(self, line, record_line): record_line += line @@ -188,7 +185,13 @@ class MT940(object): """skip header lines, create current statement""" for dummy_i in range(self.header_lines): iterator.next() - self.current_statement = BankStatement() + self.current_statement = { + 'name': None, + 'date': None, + 'balance_start': 0.0, + 'balance_end_real': 0.0, + 'transactions': [] + } def handle_footer(self, dummy_line, dummy_iterator): """add current statement to list, reset state""" @@ -199,7 +202,7 @@ class MT940(object): """find a function to handle the record represented by line""" tag_match = re.match(self.tag_regex, line) tag = tag_match.group(0).strip(':') - if not hasattr(self, 'handle_tag_%s' % tag): + if not hasattr(self, 'handle_tag_%s' % tag): # pragma: no cover logging.error('Unknown tag %s', tag) logging.error(line) return @@ -213,7 +216,7 @@ class MT940(object): def handle_tag_25(self, data): """Handle tag 25: local bank account information.""" data = data.replace('EUR', '').replace('.', '').strip() - self.current_statement.local_account = data + self.account_number = data def handle_tag_28C(self, data): """Sequence number within batch - normally only zeroes.""" @@ -224,18 +227,21 @@ class MT940(object): # For the moment only first 60F record # The alternative would be to split the file and start a new # statement for each 20: tag encountered. - stmt = self.current_statement - if not stmt.local_currency: - stmt.local_currency = data[7:10] - stmt.start_balance = str2amount(data[0], data[10:]) + if not self.currency_code: + self.currency_code = data[7:10] + self.current_statement['balance_start'] = str2amount( + data[0], + data[10:] + ) def handle_tag_61(self, data): """get transaction values""" - transaction = self.current_statement.create_transaction() - self.current_transaction = transaction - transaction.execution_date = datetime.strptime(data[:6], '%y%m%d') - transaction.value_date = datetime.strptime(data[:6], '%y%m%d') - # ...and the rest already is highly bank dependent + self.current_statement['transactions'].append({}) + self.current_transaction = self.current_statement['transactions'][-1] + self.current_transaction['date'] = datetime.strptime( + data[:6], + '%y%m%d' + ) def handle_tag_62F(self, data): """Get ending balance, statement date and id. @@ -251,19 +257,24 @@ class MT940(object): Depending on the bank, there might be multiple 62F tags in the import file. The last one counts. """ - stmt = self.current_statement - stmt.end_balance = str2amount(data[0], data[10:]) - stmt.date = datetime.strptime(data[1:7], '%y%m%d') + + self.current_statement['balance_end_real'] = str2amount( + data[0], + data[10:] + ) + self.current_statement['date'] = datetime.strptime(data[1:7], '%y%m%d') + # Only replace logically empty (only whitespace or zeroes) id's: # But do replace statement_id's added before (therefore starting # with local_account), because we need the date on the last 62F # record. - test_empty_id = re.sub(r'[\s0]', '', stmt.statement_id) - if ((not test_empty_id) or - (stmt.statement_id.startswith(stmt.local_account))): - stmt.statement_id = '%s-%s' % ( - stmt.local_account, - stmt.date.strftime('%Y-%m-%d'), + statement_name = self.current_statement['name'] or '' + test_empty_id = re.sub(r'[\s0]', '', statement_name) + is_account_number = statement_name.startswith(self.account_number) + if ((not test_empty_id) or is_account_number): + self.current_statement['name'] = '%s-%s' % ( + self.account_number, + self.current_statement['date'].strftime('%Y-%m-%d'), ) def handle_tag_64(self, data): diff --git a/account_bank_statement_import_mt940_base/test_files/test-ing.940 b/account_bank_statement_import_mt940_base/test_files/test-ing.940 new file mode 100644 index 0000000..5d98ef7 --- /dev/null +++ b/account_bank_statement_import_mt940_base/test_files/test-ing.940 @@ -0,0 +1,61 @@ +{1:F01INGBNL2ABXXX0000000000} +{2:I940INGBNL2AXXXN} +{4: +:20:P140220000000001 +:25:NL77INGB0574908765EUR +:28C:0000 +0 +:60F:C140219EUR662,23 +:61:1402200220C1,56NTRFEREF//00000000001 +005 +/TRCD/00100/ +:86:/EREF/EV12341REP1231456T1234//CNTP/NL32INGB0000012345/INGBNL2 +A/ING BANK NV INZAKE WEB///REMI/USTD//EV10001REP1000000T1000/ +:61:1402200220D1,57NTRFPREF//00000000001006 +/TRCD/00200/ +:86:/PREF/M000000003333333//REMI/USTD//TOTAAL 1 VZ/ +:61:1402200220C1,57NRTIEREF//00000000001007 +/TRCD/00190/ +:86:/RTRN/MS03//EREF/20120123456789//CNTP/NL32INGB0000012345/INGB +NL2A/J.Janssen///REMI/USTD//Factuurnr 123456 Klantnr 00123/ +:61:1402200220D1,14NDDTEREF//00000000001009 +/TRCD/010 +16 +/ +:86:/EREF/EV123R +EP123412T1234//MARF/MND +- +EV01//CSID/NL32ZZZ9999999 +91234//CNTP/NL32INGB0000012345/INGBNL2A/ING Bank N.V. inzake WeB/ +//REMI/USTD//EV123REP123412T1234/ +:61:1402200220C1,45NDDTPREF//00000000001008 +/TRCD/01000/ +:86:/PREF/M000000001111111/ +/CSID/ +NL32ZZZ999999991234 +/ +/REMI/USTD// +TOTAAL 1 POSTEN/ +:61:1402200220D12,75NRTIEREF//00000000001010 +/TRCD/01315/ +:86:/RTRN/MS03//EREF/20120501P0123478//MARF/MND +- +120123//CSID/NL32 +ZZZ999999991234//CNTP/NL32INGB0000012345/INGBNL2A/J.Janssen///REM +I/USTD//CO +NTRIBUTIE FEB 2014/ +:61:1402200220C32,00NTRF9001123412341234//00000000001011 +/TRCD/00108/ +:86:/EREF/15814016000676480//CNTP/NL32INGB0000012345/INGBNL2A/J.J +anssen///REMI/STRD/CUR/9001123412341234/ +:61:1402200220D119,00NTRF1070123412341234//00000000001012 +/ +TRCD/00108/ +:86:/EREF/15614016000384600//CNTP/NL32INGB0000012345/INGBNL2A/ING +BANK NV///REMI/STRD/CUR/1070123412341234/ +:62F:C140220EUR564,35 +:64:C140220EUR564,35 +:65:C140221EUR564,35 +:65:C140224EUR564,35 +:86:/SUM/4/4/134,46/36,58/ +-} diff --git a/account_bank_statement_import_mt940_base/test_files/test-rabo.swi b/account_bank_statement_import_mt940_base/test_files/test-rabo.swi new file mode 100644 index 0000000..6e347e2 --- /dev/null +++ b/account_bank_statement_import_mt940_base/test_files/test-rabo.swi @@ -0,0 +1,29 @@ +:940: +:20:940S140102 +:25:NL34RABO0142623393 EUR +:28C:0 +:60F:C131231EUR000000004433,52 +:61:140102C000000000400,00N541NONREF +NL66RABO0160878799 +:86:/ORDP//NAME/R. SMITH/ADDR/Green market 74 3311BE Sheepcity Nederl +and NL/REMI/Test money paid by other partner: +/ISDT/2014-01-02 +:62F:C140102EUR000000004833,52 +:20:940S140103 +:25:NL34RABO0142623393 EUR +:28C:0 +:60F:C140102EUR000000004833,52 +:62F:C140103EUR000000004833,52 +:20:940S140106 +:25:NL34RABO0142623393 EUR +:28C:0 +:60F:C140103EUR000000004833,52 +:61:140101D000000000034,61N093NONREF +:86:/BENM//NAME/Kosten/REMI/Periode 01-10-2013 t/m 31-12-2013/ISDT/20 +14-01-01 +:62F:C140106EUR000000004798,91 +:20:940S140107 +:25:NL34RABO0142623393 EUR +:28C:0 +:60F:C140106EUR000000004798,91 +:62F:C140107EUR000000004798,91 diff --git a/account_bank_statement_import_mt940_base/tests/__init__.py b/account_bank_statement_import_mt940_base/tests/__init__.py new file mode 100644 index 0000000..bb3456a --- /dev/null +++ b/account_bank_statement_import_mt940_base/tests/__init__.py @@ -0,0 +1 @@ +from . import test_import_bank_statement diff --git a/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py b/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py new file mode 100644 index 0000000..b25ffc9 --- /dev/null +++ b/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Onestein () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import base64 +from mock import patch +from odoo.tests.common import TransactionCase +from odoo.modules.module import get_module_resource +from ..mt940 import MT940, get_subfields, handle_common_subfields + + +class TestImport(TransactionCase): + """Run test to import mt940 import.""" + transactions = [ + { + 'account_number': 'NL46ABNA0499998748', + 'amount': -754.25, + 'ref': '435005714488-ABNO33052620', + 'name': 'test line', + }, + ] + + def setUp(self): + super(TestImport, self).setUp() + bank1 = self.env['res.partner.bank'].create({ + 'acc_number': 'NL77INGB0574908765', + 'partner_id': self.env.ref('base.main_partner').id, + 'company_id': self.env.ref('base.main_company').id, + 'bank_id': self.env.ref('base.res_bank_1').id, + }) + self.env['account.journal'].create({ + 'name': 'Bank Journal - (test1 mt940)', + 'code': 'TBNK1MT940', + 'type': 'bank', + 'bank_account_id': bank1.id, + 'currency_id': self.env.ref('base.EUR').id, + }) + bank2 = self.env['res.partner.bank'].create({ + 'acc_number': 'NL34RABO0142623393', + 'partner_id': self.env.ref('base.main_partner').id, + 'company_id': self.env.ref('base.main_company').id, + 'bank_id': self.env.ref('base.res_bank_1').id, + }) + self.env['account.journal'].create({ + 'name': 'Bank Journal - (test2 mt940)', + 'code': 'TBNK2MT940', + 'type': 'bank', + 'bank_account_id': bank2.id, + 'currency_id': self.env.ref('base.EUR').id, + }) + + self.data =\ + "/BENM//NAME/Cost/REMI/Period 01-10-2013 t/m 31-12-2013/ISDT/20" + self.codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', 'ISDT', 'REMI'] + + def test_statement_import(self): + """Test correct creation of single statement.""" + + def _prepare_statement_lines(statements): + transact = self.transactions[0] + for st_vals in statements[2]: + for line_vals in st_vals['transactions']: + line_vals['amount'] = transact['amount'] + line_vals['name'] = transact['name'] + line_vals['account_number'] = transact['account_number'] + line_vals['ref'] = transact['ref'] + + testfile = get_module_resource( + 'account_bank_statement_import_mt940_base', + 'test_files', + 'test-ing.940', + ) + parser = MT940() + datafile = open(testfile, 'rb').read() + statements = parser.parse(datafile) + + _prepare_statement_lines(statements) + + path_addon = 'odoo.addons.account_bank_statement_import.' + path_file = 'account_bank_statement_import.' + path_class = 'AccountBankStatementImport.' + method = path_addon + path_file + path_class + '_parse_file' + with patch(method) as my_mock: + my_mock.return_value = statements + + action = self.env['account.bank.statement.import'].create({ + 'data_file': base64.b64encode(datafile), + }).import_file() + + transact = self.transactions[0] + for statement in self.env['account.bank.statement'].browse( + action['context']['statement_ids'] + ): + for line in statement.line_ids: + self.assertTrue( + line.bank_account_id.acc_number == + transact['account_number']) + self.assertTrue(line.amount == transact['amount']) + self.assertTrue(line.date == '2014-02-20') + self.assertTrue(line.name == transact['name']) + self.assertTrue(line.ref == transact['ref']) + + def test_get_subfields(self): + """Unit Test function get_subfields().""" + + res = get_subfields(self.data, self.codewords) + espected_res = { + 'BENM': [''], + 'NAME': ['Cost'], + 'REMI': ['Period 01-10-2013 t', 'm 31-12-2013'], + 'ISDT': ['20'], + } + self.assertTrue(res == espected_res) + + def test_handle_common_subfields(self): + """Unit Test function handle_common_subfields().""" + + subfields = get_subfields(self.data, self.codewords) + transaction = self.transactions[0] + + handle_common_subfields(transaction, subfields) + + def test_statement_import2(self): + """Test correct creation of single statement.""" + + def _prepare_statement_lines(statements): + transact = self.transactions[0] + for st_vals in statements[2]: + for line_vals in st_vals['transactions']: + line_vals['amount'] = transact['amount'] + line_vals['name'] = transact['name'] + line_vals['account_number'] = transact['account_number'] + line_vals['ref'] = transact['ref'] + + testfile = get_module_resource( + 'account_bank_statement_import_mt940_base', + 'test_files', + 'test-rabo.swi', + ) + parser = MT940() + parser.header_regex = '^:940:' # Start of header + parser.header_lines = 1 # Number of lines to skip + datafile = open(testfile, 'rb').read() + statements = parser.parse(datafile) + + _prepare_statement_lines(statements) + + path_addon = 'odoo.addons.account_bank_statement_import.' + path_file = 'account_bank_statement_import.' + path_class = 'AccountBankStatementImport.' + method = path_addon + path_file + path_class + '_parse_file' + with patch(method) as my_mock: + my_mock.return_value = statements + + action = self.env['account.bank.statement.import'].create({ + 'data_file': base64.b64encode(datafile), + }).import_file() + + transact = self.transactions[0] + for statement in self.env['account.bank.statement'].browse( + action['context']['statement_ids'] + ): + for line in statement.line_ids: + self.assertTrue( + line.bank_account_id.acc_number == + transact['account_number']) + self.assertTrue(line.amount == transact['amount']) + self.assertTrue(line.date) + self.assertTrue(line.name == transact['name']) + self.assertTrue(line.ref == transact['ref']) From 0a18da1c6ce5644652d51a96c1002c7130d20b6c Mon Sep 17 00:00:00 2001 From: Andrea Date: Sat, 13 Jan 2018 14:02:02 +0100 Subject: [PATCH 6/6] Backport of V11 developments (from #137) --- .../README.rst | 5 +- .../__manifest__.py | 20 +-- .../mt940.py | 121 ++++++++++-------- .../test_files/test-sns.940 | 78 +++++++++++ .../test_files/test-wrong-file.940 | 11 ++ .../tests/test_import_bank_statement.py | 88 +++++++++++-- 6 files changed, 243 insertions(+), 80 deletions(-) create mode 100755 account_bank_statement_import_mt940_base/test_files/test-sns.940 create mode 100644 account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 diff --git a/account_bank_statement_import_mt940_base/README.rst b/account_bank_statement_import_mt940_base/README.rst index 2571371..86c524f 100644 --- a/account_bank_statement_import_mt940_base/README.rst +++ b/account_bank_statement_import_mt940_base/README.rst @@ -1,5 +1,5 @@ -.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg - :target: http://www.gnu.org/licenses/agpl +.. image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: https://www.gnu.org/licenses/agpl :alt: License: AGPL-3 ==================== @@ -36,6 +36,7 @@ Contributors * Stefan Rijnhart * Ronald Portier * Andrea Stirpe +* Fekete Mihai Maintainer ---------- diff --git a/account_bank_statement_import_mt940_base/__manifest__.py b/account_bank_statement_import_mt940_base/__manifest__.py index 212734b..7569c40 100644 --- a/account_bank_statement_import_mt940_base/__manifest__.py +++ b/account_bank_statement_import_mt940_base/__manifest__.py @@ -1,22 +1,6 @@ # -*- coding: utf-8 -*- -############################################################################## -# -# Copyright (C) 2013-2015 Therp BV -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## +# Copyright (C) 2013-2015 Therp BV + { 'name': 'MT940 Bank Statements Import', 'version': '10.0.1.0.0', diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 24a6969..9bd571f 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -1,23 +1,7 @@ -# -*- coding: utf-8 -*- +# Copyright (C) 2014-2015 Therp BV . +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). """Generic parser for MT940 files, base for customized versions per bank.""" -############################################################################## -# -# Copyright (C) 2014-2015 Therp BV . -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## + import re import logging from datetime import datetime @@ -138,35 +122,65 @@ class MT940(object): (line[:12], self.mt940_type) ) - def parse(self, data): + def is_mt940_statement(self, line): + """determine if line is the start of a statement""" + if not bool(line.startswith('{4:')): + raise ValueError( + 'The pre processed match %s does not seem to be a' + ' valid %s MT940 format bank statement. Every statement' + ' should start be a dict starting with {4:..' % line + ) + + def pre_process_data(self, data): + matches = [] + self.is_mt940(line=data) + data = data.replace( + '-}', '}').replace('}{', '}\r\n{').replace('\r\n', '\n') + if data.startswith(':940:'): + for statement in data.replace(':940:', '').split(':20:'): + match = '{4:\n:20:' + statement + '}' + matches.append(match) + else: + tag_re = re.compile( + r'(\{4:[^{}]+\})', + re.MULTILINE) + matches = tag_re.findall(data) + return matches + + def parse(self, data, header_lines=None): """Parse mt940 bank statement file contents.""" - self.is_mt940(data) - iterator = data.replace('\r\n', '\n').split('\n').__iter__() - line = None - record_line = '' - try: - while True: - if not self.current_statement: - self.handle_header(line, iterator) - line = iterator.next() - if not self.is_tag(line) and not self.is_footer(line): - record_line = self.add_record_line(line, record_line) - continue + data = data.decode() + matches = self.pre_process_data(data) + for match in matches: + self.is_mt940_statement(line=match) + iterator = '\n'.join( + match.split('\n')[1:-1]).split('\n').__iter__() + line = None + record_line = '' + try: + while True: + if not self.current_statement: + self.handle_header(line, iterator, + header_lines=header_lines) + line = iterator.next() + if not self.is_tag(line) and not self.is_footer(line): + record_line = self.add_record_line(line, record_line) + continue + if record_line: + self.handle_record(record_line) + if self.is_footer(line): + self.handle_footer(line, iterator) + record_line = '' + continue + record_line = line + except StopIteration: + pass + if self.current_statement: if record_line: self.handle_record(record_line) - if self.is_footer(line): - self.handle_footer(line, iterator) record_line = '' - continue - record_line = line - except StopIteration: - pass - if self.current_statement: - if record_line: - self.handle_record(record_line) - record_line = '' - self.statements.append(self.current_statement) - self.current_statement = None + self.statements.append(self.current_statement) + self.current_statement = None return self.currency_code, self.account_number, self.statements def add_record_line(self, line, record_line): @@ -181,9 +195,11 @@ class MT940(object): """determine if a line has a tag""" return line and bool(re.match(self.tag_regex, line)) - def handle_header(self, dummy_line, iterator): + def handle_header(self, dummy_line, iterator, header_lines=None): """skip header lines, create current statement""" - for dummy_i in range(self.header_lines): + if not header_lines: + header_lines = self.header_lines + for dummy_i in range(header_lines): iterator.next() self.current_statement = { 'name': None, @@ -229,10 +245,13 @@ class MT940(object): # statement for each 20: tag encountered. if not self.currency_code: self.currency_code = data[7:10] - self.current_statement['balance_start'] = str2amount( - data[0], - data[10:] - ) + self.current_statement['balance_start'] = str2amount( + data[0], + data[10:] + ) + if not self.current_statement['date']: + self.current_statement['date'] = datetime.strptime(data[1:7], + '%y%m%d') def handle_tag_61(self, data): """get transaction values""" @@ -271,7 +290,7 @@ class MT940(object): statement_name = self.current_statement['name'] or '' test_empty_id = re.sub(r'[\s0]', '', statement_name) is_account_number = statement_name.startswith(self.account_number) - if ((not test_empty_id) or is_account_number): + if not test_empty_id or is_account_number: self.current_statement['name'] = '%s-%s' % ( self.account_number, self.current_statement['date'].strftime('%Y-%m-%d'), diff --git a/account_bank_statement_import_mt940_base/test_files/test-sns.940 b/account_bank_statement_import_mt940_base/test_files/test-sns.940 new file mode 100755 index 0000000..7cdf74a --- /dev/null +++ b/account_bank_statement_import_mt940_base/test_files/test-sns.940 @@ -0,0 +1,78 @@ +{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4: +:20:0000000000 +:25:NL05SNSB0908244436 +:28C:361/1 +:60F:C171227EUR3026,96 +:61:1712271227D713,13NOVBNL49RABO0166416932 +gerrits glas en schilderwerk +:86:NL49RABO0166416932 gerrits glas en schilderwerk + +Factuur 17227/248/20 + + + +:61:1712271227D10,18NBEA//228ohu/972795 +:86: + +Jumbo Wijchen B.V. >WIJCHEN 27.12.2017 13U38 KV005 4XZJ4Z M +CC:5411 Contactloze betaling + + +:61:1712271227D13,52NINCNL94INGB0000869000 +vitens nv +:86:NL94INGB0000869000 vitens nv + +Europese incasso door:VITENS NV NL-Factuurnr 072304597540 VNKlant +nr 00000000000 BTW 0,77PC 6605 DW 6223, DEC-Incassant ID: NL84ZZZ0 +50695810000-Kenmerk Machtiging: bla + +:61:1712271227D25,61NBEA//229hro/195867 +:86: + +Albert Heijn 1370 >WIJCHEN 27.12.2017 19U13 KV006 70X708 M +CC:5411 + + +:62F:C171227EUR2264,52 +-}{5:} +{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4: +:20:0000000000 +:25:NL05SNSB0908244436 +:28C:362/1 +:60F:C171228EUR2264,52 +:61:1712281228D10,95NINCNL40RABO0127859497 +antagonist b.v. +:86:NL40RABO0127859497 antagonist b.v. + +Europese incasso door:ANTAGONIST B.V. NL-DDWPN954156 AI529942Cweb +share.nl-Incassant ID: NL65ZZZ091364410000-Kenmerk Machtiging: bla + +:61:1712281228C0,00NDIV +:86: + +Mobiel betalen is vanaf nu niet meer mogelijk voor B +ERG M A VAN DEN met Lenovo P2 + + +:62F:C171228EUR2253,57 +-}{5:} +{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4: +:20:0000000000 +:25:NL05SNSB0908244436 +:28C:363/1 +:60F:C171229EUR2253,57 +:61:1712291229D907,29NINCNL19ABNA0427093546 +florius +:86:NL19ABNA0427093546 florius + +Europese incasso door:FLORIUS NL-Verschuldigde bedragen PERIODE 1 +2-2017-Incassant ID: NL42ZZZ080242850000-Kenmerk Machtiging: bla + +:61:1712291229D35,00NINCNL28DEUT0265186439 +stichting derdengelden bucka +:86:NL28DEUT0265186439 stichting derdengelden bucka + +Europese incasso door:STICHTING DERDENGELDEN BUCKAROO-T-Mobile Th +uis B.V.: Rekening dec 2017. Bekijk je rekening op thuismy.t-mobi +le.nl-Incassant ID: NL39ZZZ302317620000-Kenmerk Machtiging: bla +-}{5:} diff --git a/account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 b/account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 new file mode 100644 index 0000000..ae75738 --- /dev/null +++ b/account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 @@ -0,0 +1,11 @@ +:9401: +:20:940S140102 +:25:NL34RABO0142623393 EUR +:28C:0 +:60F:C131231EUR000000004433,52 +:61:140102C000000000400,00N541NONREF +NL66RABO0160878799 +:86:/ORDP//NAME/R. SMITH/ADDR/Green market 74 3311BE Sheepcity Nederl +and NL/REMI/Test money paid by other partner: +/ISDT/2014-01-02 +:62F:C140102EUR000000004833,52 diff --git a/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py b/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py index b25ffc9..bae5ca5 100644 --- a/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py +++ b/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py @@ -48,13 +48,38 @@ class TestImport(TransactionCase): 'bank_account_id': bank2.id, 'currency_id': self.env.ref('base.EUR').id, }) + bank3 = self.env['res.partner.bank'].create({ + 'acc_number': 'NL05SNSB0908244436', + 'partner_id': self.env.ref('base.main_partner').id, + 'company_id': self.env.ref('base.main_company').id, + 'bank_id': self.env.ref('base.res_bank_1').id, + }) + self.env['account.journal'].create({ + 'name': 'Bank Journal - (test3 mt940)', + 'code': 'TBNK3MT940', + 'type': 'bank', + 'bank_account_id': bank3.id, + 'currency_id': self.env.ref('base.EUR').id, + }) self.data =\ "/BENM//NAME/Cost/REMI/Period 01-10-2013 t/m 31-12-2013/ISDT/20" self.codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', 'ISDT', 'REMI'] + def test_wrong_file_import(self): + """Test wrong file import.""" + testfile = get_module_resource( + 'account_bank_statement_import_mt940_base', + 'test_files', + 'test-wrong-file.940', + ) + parser = MT940() + datafile = open(testfile, 'rb').read() + with self.assertRaises(ValueError): + parser.parse(datafile, header_lines=1) + def test_statement_import(self): - """Test correct creation of single statement.""" + """Test correct creation of single statement ING.""" def _prepare_statement_lines(statements): transact = self.transactions[0] @@ -72,7 +97,7 @@ class TestImport(TransactionCase): ) parser = MT940() datafile = open(testfile, 'rb').read() - statements = parser.parse(datafile) + statements = parser.parse(datafile, header_lines=1) _prepare_statement_lines(statements) @@ -89,8 +114,7 @@ class TestImport(TransactionCase): transact = self.transactions[0] for statement in self.env['account.bank.statement'].browse( - action['context']['statement_ids'] - ): + action['context']['statement_ids']): for line in statement.line_ids: self.assertTrue( line.bank_account_id.acc_number == @@ -121,7 +145,7 @@ class TestImport(TransactionCase): handle_common_subfields(transaction, subfields) def test_statement_import2(self): - """Test correct creation of single statement.""" + """Test correct creation of single statement RABO.""" def _prepare_statement_lines(statements): transact = self.transactions[0] @@ -141,7 +165,7 @@ class TestImport(TransactionCase): parser.header_regex = '^:940:' # Start of header parser.header_lines = 1 # Number of lines to skip datafile = open(testfile, 'rb').read() - statements = parser.parse(datafile) + statements = parser.parse(datafile, header_lines=1) _prepare_statement_lines(statements) @@ -155,11 +179,11 @@ class TestImport(TransactionCase): action = self.env['account.bank.statement.import'].create({ 'data_file': base64.b64encode(datafile), }).import_file() - + # The file contains 4 statements, but only 2 with transactions + self.assertTrue(len(action['context']['statement_ids']) == 2) transact = self.transactions[0] for statement in self.env['account.bank.statement'].browse( - action['context']['statement_ids'] - ): + action['context']['statement_ids']): for line in statement.line_ids: self.assertTrue( line.bank_account_id.acc_number == @@ -168,3 +192,49 @@ class TestImport(TransactionCase): self.assertTrue(line.date) self.assertTrue(line.name == transact['name']) self.assertTrue(line.ref == transact['ref']) + + def test_statement_import3(self): + """Test correct creation of multiple statements SNS.""" + + def _prepare_statement_lines(statements): + transact = self.transactions[0] + for st_vals in statements[2]: + for line_vals in st_vals['transactions']: + line_vals['amount'] = transact['amount'] + line_vals['name'] = transact['name'] + line_vals['account_number'] = transact['account_number'] + line_vals['ref'] = transact['ref'] + + testfile = get_module_resource( + 'account_bank_statement_import_mt940_base', + 'test_files', + 'test-sns.940', + ) + parser = MT940() + datafile = open(testfile, 'rb').read() + statements = parser.parse(datafile, header_lines=1) + + _prepare_statement_lines(statements) + + path_addon = 'odoo.addons.account_bank_statement_import.' + path_file = 'account_bank_statement_import.' + path_class = 'AccountBankStatementImport.' + method = path_addon + path_file + path_class + '_parse_file' + with patch(method) as my_mock: + my_mock.return_value = statements + + action = self.env['account.bank.statement.import'].create({ + 'data_file': base64.b64encode(datafile), + }).import_file() + self.assertTrue(len(action['context']['statement_ids']) == 3) + transact = self.transactions[-1] + for statement in self.env['account.bank.statement'].browse( + action['context']['statement_ids'][-1]): + for line in statement.line_ids: + self.assertTrue( + line.bank_account_id.acc_number == + transact['account_number']) + self.assertTrue(line.amount == transact['amount']) + self.assertTrue(line.date == statement.date) + self.assertTrue(line.name == transact['name']) + self.assertTrue(line.ref == transact['ref'])