From 69fd180172f0cf7bbddd77675345c8fc6766d020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 23 Jul 2015 15:32:38 +0200 Subject: [PATCH 01/12] [IMP] rename bank_statement_parse_ to account_bank_statement_import_ Move parserlib to account_bank_statement_import and remove bank_statement_parse module --- .../README.rst | 53 ++++ .../__init__.py | 1 + .../__openerp__.py | 31 +++ .../mt940.py | 261 ++++++++++++++++++ 4 files changed, 346 insertions(+) create mode 100644 account_bank_statement_import_mt940_base/README.rst create mode 100644 account_bank_statement_import_mt940_base/__init__.py create mode 100644 account_bank_statement_import_mt940_base/__openerp__.py create mode 100644 account_bank_statement_import_mt940_base/mt940.py diff --git a/account_bank_statement_import_mt940_base/README.rst b/account_bank_statement_import_mt940_base/README.rst new file mode 100644 index 0000000..73a280a --- /dev/null +++ b/account_bank_statement_import_mt940_base/README.rst @@ -0,0 +1,53 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License: AGPL-3 + +Bank Statement MT940 +==================== + +This module provides a generic parser for MT940 files. Given that MT940 is a +non-open non-standard of pure evil in the way that every bank cooks up its own +interpretation of it, this addon alone won't help you much. It is rather +intended to be used by other addons to implement the dialect specific to a +certain bank. + +See bank_statement_parse_nl_ing_mt940 for an example on how to use it. + +Known issues / Roadmap +====================== + +* None + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed feedback +`here `_. + + +Credits +======= + +Contributors +------------ + +* Stefan Rijnhart +* Ronald Portier + +Maintainer +---------- + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit http://odoo-community.org. +This module should make it easy to migrate bank statement import +modules written for earlies versions of Odoo/OpenERP. diff --git a/account_bank_statement_import_mt940_base/__init__.py b/account_bank_statement_import_mt940_base/__init__.py new file mode 100644 index 0000000..d31f98c --- /dev/null +++ b/account_bank_statement_import_mt940_base/__init__.py @@ -0,0 +1 @@ +from . import mt940 diff --git a/account_bank_statement_import_mt940_base/__openerp__.py b/account_bank_statement_import_mt940_base/__openerp__.py new file mode 100644 index 0000000..760851b --- /dev/null +++ b/account_bank_statement_import_mt940_base/__openerp__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (C) 2013-2015 Therp BV +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +{ + 'name': 'MT940 Bank Statements Import', + 'version': '1.1', + 'license': 'AGPL-3', + 'author': 'Odoo Community Association (OCA), Therp BV', + 'website': 'https://github.com/OCA/bank-statement-import', + 'category': 'Banking addons', + 'depends': [ + 'account_bank_statement_import', + ], + 'installable': True +} diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py new file mode 100644 index 0000000..c410caf --- /dev/null +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -0,0 +1,261 @@ +# -*- coding: utf-8 -*- +"""Generic parser for MT940 files, base for customized versions per bank.""" +############################################################################## +# +# Copyright (C) 2014-2015 Therp BV . +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +import re +import logging +from datetime import datetime + +from openerp.addons.account_bank_statement_import.parserlib import BankStatement + + +def str2amount(sign, amount_str): + """Convert sign (C or D) and amount in string to signed amount (float).""" + factor = (1 if sign == 'C' else -1) + return factor * float(amount_str.replace(',', '.')) + + +def get_subfields(data, codewords): + """Return dictionary with value array for each codeword in data. + + For instance: + data = + /BENM//NAME/Kosten/REMI/Periode 01-10-2013 t/m 31-12-2013/ISDT/20 + codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', ISDT', 'REMI'] + Then return subfields = { + 'BENM': [], + 'NAME': ['Kosten'], + 'REMI': ['Periode 01-10-2013 t', 'm 31-12-2013'], + 'ISDT': ['20'], + } + """ + subfields = {} + current_codeword = None + for word in data.split('/'): + if not word and not current_codeword: + continue + if word in codewords: + current_codeword = word + subfields[current_codeword] = [] + continue + if current_codeword in subfields: + subfields[current_codeword].append(word) + return subfields + + +def get_counterpart(transaction, subfield): + """Get counterpart from transaction. + + Counterpart is often stored in subfield of tag 86. The subfield + can be BENM, ORDP, CNTP""" + if not subfield: + return # subfield is empty + if len(subfield) >= 1 and subfield[0]: + transaction.remote_account = subfield[0] + if len(subfield) >= 2 and subfield[1]: + transaction.remote_bank_bic = subfield[1] + if len(subfield) >= 3 and subfield[2]: + transaction.remote_owner = subfield[2] + if len(subfield) >= 4 and subfield[3]: + transaction.remote_owner_city = subfield[3] + + +def handle_common_subfields(transaction, subfields): + """Deal with common functionality for tag 86 subfields.""" + # Get counterpart from CNTP, BENM or ORDP subfields: + for counterpart_field in ['CNTP', 'BENM', 'ORDP']: + if counterpart_field in subfields: + get_counterpart(transaction, subfields[counterpart_field]) + # REMI: Remitter information (text entered by other party on trans.): + if 'REMI' in subfields: + transaction.message = ( + '/'.join(x for x in subfields['REMI'] if x)) + # Get transaction reference subfield (might vary): + if transaction.eref in subfields: + transaction.eref = ''.join( + subfields[transaction.eref]) + + +class MT940(object): + """Inherit this class in your account_banking.parsers.models.parser, + define functions to handle the tags you need to handle and adjust static + variables as needed. + + At least, you should override handle_tag_61 and handle_tag_86. + Don't forget to call super. + + handle_tag_* functions receive the remainder of the the line (that is, + without ':XX:') and are supposed to write into self.current_transaction + """ + + def __init__(self): + """Initialize parser - override at least header_regex. + + This in fact uses the ING syntax, override in others.""" + self.mt940_type = 'General' + self.header_lines = 3 # Number of lines to skip + self.header_regex = '^0000 01INGBNL2AXXXX|^{1' + self.footer_regex = '^-}$|^-XXX$' # Stop processing on seeing this + self.tag_regex = '^:[0-9]{2}[A-Z]*:' # Start of new tag + self.current_statement = None + self.current_transaction = None + self.statements = [] + + def is_mt940(self, line): + """determine if a line is the header of a statement""" + if not bool(re.match(self.header_regex, line)): + raise ValueError( + 'File starting with %s does not seem to be a' + ' valid %s MT940 format bank statement.' % + (line[:12], self.mt940_type) + ) + + def parse(self, data): + """Parse mt940 bank statement file contents.""" + self.is_mt940(data) + iterator = data.replace('\r\n', '\n').split('\n').__iter__() + line = None + record_line = '' + try: + while True: + if not self.current_statement: + self.handle_header(line, iterator) + line = iterator.next() + if not self.is_tag(line) and not self.is_footer(line): + record_line += line + continue + if record_line: + self.handle_record(record_line) + if self.is_footer(line): + self.handle_footer(line, iterator) + record_line = '' + continue + record_line = line + except StopIteration: + pass + if self.current_statement: + if record_line: + self.handle_record(record_line) + record_line = '' + self.statements.append(self.current_statement) + self.current_statement = None + return self.statements + + def is_footer(self, line): + """determine if a line is the footer of a statement""" + return line and bool(re.match(self.footer_regex, line)) + + def is_tag(self, line): + """determine if a line has a tag""" + return line and bool(re.match(self.tag_regex, line)) + + def handle_header(self, dummy_line, iterator): + """skip header lines, create current statement""" + for dummy_i in range(self.header_lines): + iterator.next() + self.current_statement = BankStatement() + + def handle_footer(self, dummy_line, dummy_iterator): + """add current statement to list, reset state""" + self.statements.append(self.current_statement) + self.current_statement = None + + def handle_record(self, line): + """find a function to handle the record represented by line""" + tag_match = re.match(self.tag_regex, line) + tag = tag_match.group(0).strip(':') + if not hasattr(self, 'handle_tag_%s' % tag): + logging.error('Unknown tag %s', tag) + logging.error(line) + return + handler = getattr(self, 'handle_tag_%s' % tag) + handler(line[tag_match.end():]) + + def handle_tag_20(self, data): + """Contains unique ? message ID""" + pass + + def handle_tag_25(self, data): + """Handle tag 25: local bank account information.""" + data = data.replace('EUR', '').replace('.', '').strip() + self.current_statement.local_account = data + + def handle_tag_28C(self, data): + """Sequence number within batch - normally only zeroes.""" + pass + + def handle_tag_60F(self, data): + """get start balance and currency""" + # For the moment only first 60F record + # The alternative would be to split the file and start a new + # statement for each 20: tag encountered. + stmt = self.current_statement + if not stmt.local_currency: + stmt.local_currency = data[7:10] + stmt.start_balance = str2amount(data[0], data[10:]) + + def handle_tag_61(self, data): + """get transaction values""" + transaction = self.current_statement.create_transaction() + self.current_transaction = transaction + transaction.execution_date = datetime.strptime(data[:6], '%y%m%d') + transaction.value_date = datetime.strptime(data[:6], '%y%m%d') + # ...and the rest already is highly bank dependent + + def handle_tag_62F(self, data): + """Get ending balance, statement date and id. + + We use the date on the last 62F tag as statement date, as the date + on the 60F record (previous end balance) might contain a date in + a previous period. + + We generate the statement.id from the local_account and the end-date, + this should normally be unique, provided there is a maximum of + one statement per day. + + Depending on the bank, there might be multiple 62F tags in the import + file. The last one counts. + """ + stmt = self.current_statement + stmt.end_balance = str2amount(data[0], data[10:]) + stmt.date = datetime.strptime(data[1:7], '%y%m%d') + # Only replace logically empty (only whitespace or zeroes) id's: + # But do replace statement_id's added before (therefore starting + # with local_account), because we need the date on the last 62F + # record. + test_empty_id = re.sub(r'[\s0]', '', stmt.statement_id) + if ((not test_empty_id) or + (stmt.statement_id.startswith(stmt.local_account))): + stmt.statement_id = '%s-%s' % ( + stmt.local_account, + stmt.date.strftime('%Y-%m-%d'), + ) + + def handle_tag_64(self, data): + """get current balance in currency""" + pass + + def handle_tag_65(self, data): + """get future balance in currency""" + pass + + def handle_tag_86(self, data): + """details for previous transaction, here most differences between + banks occur""" + pass From 0524ab103238abc18adaee3bc02b1f91f154233d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 23 Jul 2015 16:30:44 +0200 Subject: [PATCH 02/12] [FIX] pep8 --- account_bank_statement_import_mt940_base/mt940.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index c410caf..8677bc7 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -22,7 +22,8 @@ import re import logging from datetime import datetime -from openerp.addons.account_bank_statement_import.parserlib import BankStatement +from openerp.addons.account_bank_statement_import.parserlib import ( + BankStatement) def str2amount(sign, amount_str): From 9f7fcdd75b1fc908ab7e12afb5b1d4f00a37cd84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 9 Oct 2015 09:59:36 +0200 Subject: [PATCH 03/12] [UPD] prefix versions with 8.0 --- account_bank_statement_import_mt940_base/__openerp__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/account_bank_statement_import_mt940_base/__openerp__.py b/account_bank_statement_import_mt940_base/__openerp__.py index 760851b..6cef8dc 100644 --- a/account_bank_statement_import_mt940_base/__openerp__.py +++ b/account_bank_statement_import_mt940_base/__openerp__.py @@ -19,7 +19,7 @@ ############################################################################## { 'name': 'MT940 Bank Statements Import', - 'version': '1.1', + 'version': '8.0.1.1.0', 'license': 'AGPL-3', 'author': 'Odoo Community Association (OCA), Therp BV', 'website': 'https://github.com/OCA/bank-statement-import', From 65bfa28995d30e31edc42868f529a404bfb16cb8 Mon Sep 17 00:00:00 2001 From: "Pedro M. Baeza" Date: Wed, 14 Oct 2015 02:19:40 +0200 Subject: [PATCH 04/12] [MIG] Make modules uninstallable --- account_bank_statement_import_mt940_base/__openerp__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/account_bank_statement_import_mt940_base/__openerp__.py b/account_bank_statement_import_mt940_base/__openerp__.py index 6cef8dc..5aee03c 100644 --- a/account_bank_statement_import_mt940_base/__openerp__.py +++ b/account_bank_statement_import_mt940_base/__openerp__.py @@ -27,5 +27,5 @@ 'depends': [ 'account_bank_statement_import', ], - 'installable': True + 'installable': False } From 1d26126c187e6ffd992a4bfa2c4748fc7a861356 Mon Sep 17 00:00:00 2001 From: "Pedro M. Baeza" Date: Thu, 6 Oct 2016 14:47:55 +0200 Subject: [PATCH 05/12] [MIG] Rename manifest files --- .../{__openerp__.py => __manifest__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename account_bank_statement_import_mt940_base/{__openerp__.py => __manifest__.py} (100%) diff --git a/account_bank_statement_import_mt940_base/__openerp__.py b/account_bank_statement_import_mt940_base/__manifest__.py similarity index 100% rename from account_bank_statement_import_mt940_base/__openerp__.py rename to account_bank_statement_import_mt940_base/__manifest__.py From 41d3b42e9cd6612c35eb1e5ec9be4624787f474b Mon Sep 17 00:00:00 2001 From: "Ronald Portier (Therp BV)" Date: Sat, 13 Jun 2015 01:20:18 +0200 Subject: [PATCH 06/12] [ENH] Support zip files. --- account_bank_statement_import_mt940_base/mt940.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 8677bc7..9ddd9ce 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -111,7 +111,7 @@ class MT940(object): This in fact uses the ING syntax, override in others.""" self.mt940_type = 'General' self.header_lines = 3 # Number of lines to skip - self.header_regex = '^0000 01INGBNL2AXXXX|^{1' + self.header_regex = '^0000 01INGBNL2AXXXX|^{1' # Start of header self.footer_regex = '^-}$|^-XXX$' # Stop processing on seeing this self.tag_regex = '^:[0-9]{2}[A-Z]*:' # Start of new tag self.current_statement = None From 163c68679a148d0565c7d76d65bda4117cee2f08 Mon Sep 17 00:00:00 2001 From: "Ronald Portier (Therp BV)" Date: Fri, 2 Oct 2015 00:12:32 +0200 Subject: [PATCH 07/12] [FIX] Import exceptions.Warning as UserError. This according to OCA guidelines. Also reformat README.rst to keep lines within limit. --- account_bank_statement_import_mt940_base/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/account_bank_statement_import_mt940_base/__manifest__.py b/account_bank_statement_import_mt940_base/__manifest__.py index 5aee03c..5c672a6 100644 --- a/account_bank_statement_import_mt940_base/__manifest__.py +++ b/account_bank_statement_import_mt940_base/__manifest__.py @@ -19,7 +19,7 @@ ############################################################################## { 'name': 'MT940 Bank Statements Import', - 'version': '8.0.1.1.0', + 'version': '8.0.1.1.1', 'license': 'AGPL-3', 'author': 'Odoo Community Association (OCA), Therp BV', 'website': 'https://github.com/OCA/bank-statement-import', From ff75184cd319d20e982e106a7e3d956b8a099b9a Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Fri, 14 Oct 2016 13:32:23 +0000 Subject: [PATCH 08/12] [UPD] parse more fields as of (#71) * [UPD] parse more fields for MT940 as of https://www.ing.nl/media/ING_ming_mt940s_24_juli_tcm162-46356.pdf p 8ff * [FIX] There is no CR or LF in mt940 data to be parsed. The calling logic from the MT940 parser concatenates all lines from a mt940 tag. --- .../mt940.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 9ddd9ce..3c36f42 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -83,14 +83,28 @@ def handle_common_subfields(transaction, subfields): for counterpart_field in ['CNTP', 'BENM', 'ORDP']: if counterpart_field in subfields: get_counterpart(transaction, subfields[counterpart_field]) + if not transaction.message: + transaction.message = '' # REMI: Remitter information (text entered by other party on trans.): if 'REMI' in subfields: - transaction.message = ( - '/'.join(x for x in subfields['REMI'] if x)) + transaction.message += ( + subfields['REMI'][2] + # this might look like + # /REMI/USTD/// + # or + # /REMI/STRD/CUR// + if len(subfields['REMI']) >= 3 and subfields['REMI'][0] in [ + 'STRD', 'USTD' + ] + else + '/'.join(x for x in subfields['REMI'] if x) + ) + # EREF: End-to-end reference + if 'EREF' in subfields: + transaction.message += '/'.join(filter(bool, subfields['EREF'])) # Get transaction reference subfield (might vary): if transaction.eref in subfields: - transaction.eref = ''.join( - subfields[transaction.eref]) + transaction.eref = ''.join(subfields[transaction.eref]) class MT940(object): From 1cb6956f0f412467d4d98aa6ed0033fe09d96704 Mon Sep 17 00:00:00 2001 From: Andrea Stirpe Date: Wed, 17 May 2017 10:12:19 +0200 Subject: [PATCH 09/12] Extract method to allow extensions (#109) --- account_bank_statement_import_mt940_base/mt940.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 3c36f42..22dc5a6 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -153,7 +153,7 @@ class MT940(object): self.handle_header(line, iterator) line = iterator.next() if not self.is_tag(line) and not self.is_footer(line): - record_line += line + record_line = self.add_record_line(line, record_line) continue if record_line: self.handle_record(record_line) @@ -172,6 +172,10 @@ class MT940(object): self.current_statement = None return self.statements + def add_record_line(self, line, record_line): + record_line += line + return record_line + def is_footer(self, line): """determine if a line is the footer of a statement""" return line and bool(re.match(self.footer_regex, line)) From e62c66946963f1c0494e0062ba1695e85cf31418 Mon Sep 17 00:00:00 2001 From: Andrea Date: Tue, 12 Sep 2017 12:18:22 +0200 Subject: [PATCH 10/12] Porting to V10 --- .../README.rst | 20 ++--- .../__manifest__.py | 4 +- .../mt940.py | 81 +++++++++++-------- 3 files changed, 58 insertions(+), 47 deletions(-) diff --git a/account_bank_statement_import_mt940_base/README.rst b/account_bank_statement_import_mt940_base/README.rst index 73a280a..1a19ffc 100644 --- a/account_bank_statement_import_mt940_base/README.rst +++ b/account_bank_statement_import_mt940_base/README.rst @@ -1,6 +1,8 @@ .. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg - :alt: License: AGPL-3 + :target: http://www.gnu.org/licenses/agpl + :alt: License: AGPL-3 +==================== Bank Statement MT940 ==================== @@ -10,7 +12,7 @@ interpretation of it, this addon alone won't help you much. It is rather intended to be used by other addons to implement the dialect specific to a certain bank. -See bank_statement_parse_nl_ing_mt940 for an example on how to use it. +See account_bank_statement_import_mt940_nl_ing for an example on how to use it. Known issues / Roadmap ====================== @@ -20,11 +22,10 @@ Known issues / Roadmap Bug Tracker =========== -Bugs are tracked on `GitHub Issues `_. -In case of trouble, please check there if your issue has already been reported. -If you spotted it first, help us smashing it by providing a detailed and welcomed feedback -`here `_. - +Bugs are tracked on `GitHub Issues +`_. In case of trouble, please +check there if your issue has already been reported. If you spotted it first, +help us smash it by providing detailed and welcomed feedback. Credits ======= @@ -38,6 +39,7 @@ Contributors Maintainer ---------- + .. image:: https://odoo-community.org/logo.png :alt: Odoo Community Association :target: https://odoo-community.org @@ -48,6 +50,4 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -To contribute to this module, please visit http://odoo-community.org. -This module should make it easy to migrate bank statement import -modules written for earlies versions of Odoo/OpenERP. +To contribute to this module, please visit https://odoo-community.org. diff --git a/account_bank_statement_import_mt940_base/__manifest__.py b/account_bank_statement_import_mt940_base/__manifest__.py index 5c672a6..212734b 100644 --- a/account_bank_statement_import_mt940_base/__manifest__.py +++ b/account_bank_statement_import_mt940_base/__manifest__.py @@ -19,7 +19,7 @@ ############################################################################## { 'name': 'MT940 Bank Statements Import', - 'version': '8.0.1.1.1', + 'version': '10.0.1.0.0', 'license': 'AGPL-3', 'author': 'Odoo Community Association (OCA), Therp BV', 'website': 'https://github.com/OCA/bank-statement-import', @@ -27,5 +27,5 @@ 'depends': [ 'account_bank_statement_import', ], - 'installable': False + 'installable': True } diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 22dc5a6..6f19df8 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -22,9 +22,6 @@ import re import logging from datetime import datetime -from openerp.addons.account_bank_statement_import.parserlib import ( - BankStatement) - def str2amount(sign, amount_str): """Convert sign (C or D) and amount in string to signed amount (float).""" @@ -68,13 +65,11 @@ def get_counterpart(transaction, subfield): if not subfield: return # subfield is empty if len(subfield) >= 1 and subfield[0]: - transaction.remote_account = subfield[0] + transaction.update({'account_number': subfield[0]}) if len(subfield) >= 2 and subfield[1]: - transaction.remote_bank_bic = subfield[1] + transaction.update({'account_bic': subfield[1]}) if len(subfield) >= 3 and subfield[2]: - transaction.remote_owner = subfield[2] - if len(subfield) >= 4 and subfield[3]: - transaction.remote_owner_city = subfield[3] + transaction.update({'partner_name': subfield[2]}) def handle_common_subfields(transaction, subfields): @@ -83,11 +78,11 @@ def handle_common_subfields(transaction, subfields): for counterpart_field in ['CNTP', 'BENM', 'ORDP']: if counterpart_field in subfields: get_counterpart(transaction, subfields[counterpart_field]) - if not transaction.message: - transaction.message = '' + if not transaction.get('name'): + transaction['name'] = '' # REMI: Remitter information (text entered by other party on trans.): if 'REMI' in subfields: - transaction.message += ( + transaction['name'] += ( subfields['REMI'][2] # this might look like # /REMI/USTD/// @@ -101,10 +96,10 @@ def handle_common_subfields(transaction, subfields): ) # EREF: End-to-end reference if 'EREF' in subfields: - transaction.message += '/'.join(filter(bool, subfields['EREF'])) + transaction['name'] += '/'.join(filter(bool, subfields['EREF'])) # Get transaction reference subfield (might vary): - if transaction.eref in subfields: - transaction.eref = ''.join(subfields[transaction.eref]) + if transaction.get('ref') in subfields: + transaction['ref'] = ''.join(subfields[transaction['ref']]) class MT940(object): @@ -131,6 +126,8 @@ class MT940(object): self.current_statement = None self.current_transaction = None self.statements = [] + self.currency_code = None + self.account_number = None def is_mt940(self, line): """determine if a line is the header of a statement""" @@ -170,7 +167,7 @@ class MT940(object): record_line = '' self.statements.append(self.current_statement) self.current_statement = None - return self.statements + return self.currency_code, self.account_number, self.statements def add_record_line(self, line, record_line): record_line += line @@ -188,7 +185,13 @@ class MT940(object): """skip header lines, create current statement""" for dummy_i in range(self.header_lines): iterator.next() - self.current_statement = BankStatement() + self.current_statement = { + 'name': None, + 'date': None, + 'balance_start': 0.0, + 'balance_end_real': 0.0, + 'transactions': [] + } def handle_footer(self, dummy_line, dummy_iterator): """add current statement to list, reset state""" @@ -213,7 +216,7 @@ class MT940(object): def handle_tag_25(self, data): """Handle tag 25: local bank account information.""" data = data.replace('EUR', '').replace('.', '').strip() - self.current_statement.local_account = data + self.account_number = data def handle_tag_28C(self, data): """Sequence number within batch - normally only zeroes.""" @@ -224,18 +227,21 @@ class MT940(object): # For the moment only first 60F record # The alternative would be to split the file and start a new # statement for each 20: tag encountered. - stmt = self.current_statement - if not stmt.local_currency: - stmt.local_currency = data[7:10] - stmt.start_balance = str2amount(data[0], data[10:]) + if not self.currency_code: + self.currency_code = data[7:10] + self.current_statement['balance_start'] = str2amount( + data[0], + data[10:] + ) def handle_tag_61(self, data): """get transaction values""" - transaction = self.current_statement.create_transaction() - self.current_transaction = transaction - transaction.execution_date = datetime.strptime(data[:6], '%y%m%d') - transaction.value_date = datetime.strptime(data[:6], '%y%m%d') - # ...and the rest already is highly bank dependent + self.current_statement['transactions'].append({}) + self.current_transaction = self.current_statement['transactions'][-1] + self.current_transaction['date'] = datetime.strptime( + data[:6], + '%y%m%d' + ) def handle_tag_62F(self, data): """Get ending balance, statement date and id. @@ -251,19 +257,24 @@ class MT940(object): Depending on the bank, there might be multiple 62F tags in the import file. The last one counts. """ - stmt = self.current_statement - stmt.end_balance = str2amount(data[0], data[10:]) - stmt.date = datetime.strptime(data[1:7], '%y%m%d') + + self.current_statement['balance_end_real'] = str2amount( + data[0], + data[10:] + ) + self.current_statement['date'] = datetime.strptime(data[1:7], '%y%m%d') + # Only replace logically empty (only whitespace or zeroes) id's: # But do replace statement_id's added before (therefore starting # with local_account), because we need the date on the last 62F # record. - test_empty_id = re.sub(r'[\s0]', '', stmt.statement_id) - if ((not test_empty_id) or - (stmt.statement_id.startswith(stmt.local_account))): - stmt.statement_id = '%s-%s' % ( - stmt.local_account, - stmt.date.strftime('%Y-%m-%d'), + statement_name = self.current_statement['name'] or '' + test_empty_id = re.sub(r'[\s0]', '', statement_name) + is_account_number = statement_name.startswith(self.account_number) + if ((not test_empty_id) or is_account_number): + self.current_statement['name'] = '%s-%s' % ( + self.account_number, + self.current_statement['date'].strftime('%Y-%m-%d'), ) def handle_tag_64(self, data): From a9fc5c129513a75a2004321c52d775b1d879ffb3 Mon Sep 17 00:00:00 2001 From: Andrea Date: Mon, 6 Nov 2017 14:03:55 +0100 Subject: [PATCH 11/12] Add tests --- .../README.rst | 1 + .../mt940.py | 2 +- .../test_files/test-ing.940 | 61 +++++++ .../test_files/test-rabo.swi | 29 +++ .../tests/__init__.py | 1 + .../tests/test_import_bank_statement.py | 170 ++++++++++++++++++ 6 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 account_bank_statement_import_mt940_base/test_files/test-ing.940 create mode 100644 account_bank_statement_import_mt940_base/test_files/test-rabo.swi create mode 100644 account_bank_statement_import_mt940_base/tests/__init__.py create mode 100644 account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py diff --git a/account_bank_statement_import_mt940_base/README.rst b/account_bank_statement_import_mt940_base/README.rst index 1a19ffc..2571371 100644 --- a/account_bank_statement_import_mt940_base/README.rst +++ b/account_bank_statement_import_mt940_base/README.rst @@ -35,6 +35,7 @@ Contributors * Stefan Rijnhart * Ronald Portier +* Andrea Stirpe Maintainer ---------- diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 6f19df8..24a6969 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -202,7 +202,7 @@ class MT940(object): """find a function to handle the record represented by line""" tag_match = re.match(self.tag_regex, line) tag = tag_match.group(0).strip(':') - if not hasattr(self, 'handle_tag_%s' % tag): + if not hasattr(self, 'handle_tag_%s' % tag): # pragma: no cover logging.error('Unknown tag %s', tag) logging.error(line) return diff --git a/account_bank_statement_import_mt940_base/test_files/test-ing.940 b/account_bank_statement_import_mt940_base/test_files/test-ing.940 new file mode 100644 index 0000000..5d98ef7 --- /dev/null +++ b/account_bank_statement_import_mt940_base/test_files/test-ing.940 @@ -0,0 +1,61 @@ +{1:F01INGBNL2ABXXX0000000000} +{2:I940INGBNL2AXXXN} +{4: +:20:P140220000000001 +:25:NL77INGB0574908765EUR +:28C:0000 +0 +:60F:C140219EUR662,23 +:61:1402200220C1,56NTRFEREF//00000000001 +005 +/TRCD/00100/ +:86:/EREF/EV12341REP1231456T1234//CNTP/NL32INGB0000012345/INGBNL2 +A/ING BANK NV INZAKE WEB///REMI/USTD//EV10001REP1000000T1000/ +:61:1402200220D1,57NTRFPREF//00000000001006 +/TRCD/00200/ +:86:/PREF/M000000003333333//REMI/USTD//TOTAAL 1 VZ/ +:61:1402200220C1,57NRTIEREF//00000000001007 +/TRCD/00190/ +:86:/RTRN/MS03//EREF/20120123456789//CNTP/NL32INGB0000012345/INGB +NL2A/J.Janssen///REMI/USTD//Factuurnr 123456 Klantnr 00123/ +:61:1402200220D1,14NDDTEREF//00000000001009 +/TRCD/010 +16 +/ +:86:/EREF/EV123R +EP123412T1234//MARF/MND +- +EV01//CSID/NL32ZZZ9999999 +91234//CNTP/NL32INGB0000012345/INGBNL2A/ING Bank N.V. inzake WeB/ +//REMI/USTD//EV123REP123412T1234/ +:61:1402200220C1,45NDDTPREF//00000000001008 +/TRCD/01000/ +:86:/PREF/M000000001111111/ +/CSID/ +NL32ZZZ999999991234 +/ +/REMI/USTD// +TOTAAL 1 POSTEN/ +:61:1402200220D12,75NRTIEREF//00000000001010 +/TRCD/01315/ +:86:/RTRN/MS03//EREF/20120501P0123478//MARF/MND +- +120123//CSID/NL32 +ZZZ999999991234//CNTP/NL32INGB0000012345/INGBNL2A/J.Janssen///REM +I/USTD//CO +NTRIBUTIE FEB 2014/ +:61:1402200220C32,00NTRF9001123412341234//00000000001011 +/TRCD/00108/ +:86:/EREF/15814016000676480//CNTP/NL32INGB0000012345/INGBNL2A/J.J +anssen///REMI/STRD/CUR/9001123412341234/ +:61:1402200220D119,00NTRF1070123412341234//00000000001012 +/ +TRCD/00108/ +:86:/EREF/15614016000384600//CNTP/NL32INGB0000012345/INGBNL2A/ING +BANK NV///REMI/STRD/CUR/1070123412341234/ +:62F:C140220EUR564,35 +:64:C140220EUR564,35 +:65:C140221EUR564,35 +:65:C140224EUR564,35 +:86:/SUM/4/4/134,46/36,58/ +-} diff --git a/account_bank_statement_import_mt940_base/test_files/test-rabo.swi b/account_bank_statement_import_mt940_base/test_files/test-rabo.swi new file mode 100644 index 0000000..6e347e2 --- /dev/null +++ b/account_bank_statement_import_mt940_base/test_files/test-rabo.swi @@ -0,0 +1,29 @@ +:940: +:20:940S140102 +:25:NL34RABO0142623393 EUR +:28C:0 +:60F:C131231EUR000000004433,52 +:61:140102C000000000400,00N541NONREF +NL66RABO0160878799 +:86:/ORDP//NAME/R. SMITH/ADDR/Green market 74 3311BE Sheepcity Nederl +and NL/REMI/Test money paid by other partner: +/ISDT/2014-01-02 +:62F:C140102EUR000000004833,52 +:20:940S140103 +:25:NL34RABO0142623393 EUR +:28C:0 +:60F:C140102EUR000000004833,52 +:62F:C140103EUR000000004833,52 +:20:940S140106 +:25:NL34RABO0142623393 EUR +:28C:0 +:60F:C140103EUR000000004833,52 +:61:140101D000000000034,61N093NONREF +:86:/BENM//NAME/Kosten/REMI/Periode 01-10-2013 t/m 31-12-2013/ISDT/20 +14-01-01 +:62F:C140106EUR000000004798,91 +:20:940S140107 +:25:NL34RABO0142623393 EUR +:28C:0 +:60F:C140106EUR000000004798,91 +:62F:C140107EUR000000004798,91 diff --git a/account_bank_statement_import_mt940_base/tests/__init__.py b/account_bank_statement_import_mt940_base/tests/__init__.py new file mode 100644 index 0000000..bb3456a --- /dev/null +++ b/account_bank_statement_import_mt940_base/tests/__init__.py @@ -0,0 +1 @@ +from . import test_import_bank_statement diff --git a/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py b/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py new file mode 100644 index 0000000..b25ffc9 --- /dev/null +++ b/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Onestein () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import base64 +from mock import patch +from odoo.tests.common import TransactionCase +from odoo.modules.module import get_module_resource +from ..mt940 import MT940, get_subfields, handle_common_subfields + + +class TestImport(TransactionCase): + """Run test to import mt940 import.""" + transactions = [ + { + 'account_number': 'NL46ABNA0499998748', + 'amount': -754.25, + 'ref': '435005714488-ABNO33052620', + 'name': 'test line', + }, + ] + + def setUp(self): + super(TestImport, self).setUp() + bank1 = self.env['res.partner.bank'].create({ + 'acc_number': 'NL77INGB0574908765', + 'partner_id': self.env.ref('base.main_partner').id, + 'company_id': self.env.ref('base.main_company').id, + 'bank_id': self.env.ref('base.res_bank_1').id, + }) + self.env['account.journal'].create({ + 'name': 'Bank Journal - (test1 mt940)', + 'code': 'TBNK1MT940', + 'type': 'bank', + 'bank_account_id': bank1.id, + 'currency_id': self.env.ref('base.EUR').id, + }) + bank2 = self.env['res.partner.bank'].create({ + 'acc_number': 'NL34RABO0142623393', + 'partner_id': self.env.ref('base.main_partner').id, + 'company_id': self.env.ref('base.main_company').id, + 'bank_id': self.env.ref('base.res_bank_1').id, + }) + self.env['account.journal'].create({ + 'name': 'Bank Journal - (test2 mt940)', + 'code': 'TBNK2MT940', + 'type': 'bank', + 'bank_account_id': bank2.id, + 'currency_id': self.env.ref('base.EUR').id, + }) + + self.data =\ + "/BENM//NAME/Cost/REMI/Period 01-10-2013 t/m 31-12-2013/ISDT/20" + self.codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', 'ISDT', 'REMI'] + + def test_statement_import(self): + """Test correct creation of single statement.""" + + def _prepare_statement_lines(statements): + transact = self.transactions[0] + for st_vals in statements[2]: + for line_vals in st_vals['transactions']: + line_vals['amount'] = transact['amount'] + line_vals['name'] = transact['name'] + line_vals['account_number'] = transact['account_number'] + line_vals['ref'] = transact['ref'] + + testfile = get_module_resource( + 'account_bank_statement_import_mt940_base', + 'test_files', + 'test-ing.940', + ) + parser = MT940() + datafile = open(testfile, 'rb').read() + statements = parser.parse(datafile) + + _prepare_statement_lines(statements) + + path_addon = 'odoo.addons.account_bank_statement_import.' + path_file = 'account_bank_statement_import.' + path_class = 'AccountBankStatementImport.' + method = path_addon + path_file + path_class + '_parse_file' + with patch(method) as my_mock: + my_mock.return_value = statements + + action = self.env['account.bank.statement.import'].create({ + 'data_file': base64.b64encode(datafile), + }).import_file() + + transact = self.transactions[0] + for statement in self.env['account.bank.statement'].browse( + action['context']['statement_ids'] + ): + for line in statement.line_ids: + self.assertTrue( + line.bank_account_id.acc_number == + transact['account_number']) + self.assertTrue(line.amount == transact['amount']) + self.assertTrue(line.date == '2014-02-20') + self.assertTrue(line.name == transact['name']) + self.assertTrue(line.ref == transact['ref']) + + def test_get_subfields(self): + """Unit Test function get_subfields().""" + + res = get_subfields(self.data, self.codewords) + espected_res = { + 'BENM': [''], + 'NAME': ['Cost'], + 'REMI': ['Period 01-10-2013 t', 'm 31-12-2013'], + 'ISDT': ['20'], + } + self.assertTrue(res == espected_res) + + def test_handle_common_subfields(self): + """Unit Test function handle_common_subfields().""" + + subfields = get_subfields(self.data, self.codewords) + transaction = self.transactions[0] + + handle_common_subfields(transaction, subfields) + + def test_statement_import2(self): + """Test correct creation of single statement.""" + + def _prepare_statement_lines(statements): + transact = self.transactions[0] + for st_vals in statements[2]: + for line_vals in st_vals['transactions']: + line_vals['amount'] = transact['amount'] + line_vals['name'] = transact['name'] + line_vals['account_number'] = transact['account_number'] + line_vals['ref'] = transact['ref'] + + testfile = get_module_resource( + 'account_bank_statement_import_mt940_base', + 'test_files', + 'test-rabo.swi', + ) + parser = MT940() + parser.header_regex = '^:940:' # Start of header + parser.header_lines = 1 # Number of lines to skip + datafile = open(testfile, 'rb').read() + statements = parser.parse(datafile) + + _prepare_statement_lines(statements) + + path_addon = 'odoo.addons.account_bank_statement_import.' + path_file = 'account_bank_statement_import.' + path_class = 'AccountBankStatementImport.' + method = path_addon + path_file + path_class + '_parse_file' + with patch(method) as my_mock: + my_mock.return_value = statements + + action = self.env['account.bank.statement.import'].create({ + 'data_file': base64.b64encode(datafile), + }).import_file() + + transact = self.transactions[0] + for statement in self.env['account.bank.statement'].browse( + action['context']['statement_ids'] + ): + for line in statement.line_ids: + self.assertTrue( + line.bank_account_id.acc_number == + transact['account_number']) + self.assertTrue(line.amount == transact['amount']) + self.assertTrue(line.date) + self.assertTrue(line.name == transact['name']) + self.assertTrue(line.ref == transact['ref']) From fe73418dffdbd6d5cb94fb8312d3659093fd0579 Mon Sep 17 00:00:00 2001 From: Fekete Mihai Date: Mon, 11 Dec 2017 01:12:28 -0800 Subject: [PATCH 12/12] [11.0] [MIG] account_bank_statement_import_mt940_base. Update version to 11. Get balance start even if currency code is already set. Add SNS test, add header_lines parameter in parse file. Update code, add pre process datafile to change from different format to only one, statements as dict starting with {4:..}, split file to import multiple statements at once, the check of date of statement import not implemented, be carefully not to import twice the same statements. Update flake. Add wrong file. --- .../README.rst | 5 +- .../__manifest__.py | 23 +--- .../mt940.py | 123 ++++++++++-------- .../test_files/test-sns.940 | 78 +++++++++++ .../test_files/test-wrong-file.940 | 11 ++ .../tests/test_import_bank_statement.py | 89 +++++++++++-- 6 files changed, 245 insertions(+), 84 deletions(-) create mode 100755 account_bank_statement_import_mt940_base/test_files/test-sns.940 create mode 100644 account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 diff --git a/account_bank_statement_import_mt940_base/README.rst b/account_bank_statement_import_mt940_base/README.rst index 2571371..86c524f 100644 --- a/account_bank_statement_import_mt940_base/README.rst +++ b/account_bank_statement_import_mt940_base/README.rst @@ -1,5 +1,5 @@ -.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg - :target: http://www.gnu.org/licenses/agpl +.. image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: https://www.gnu.org/licenses/agpl :alt: License: AGPL-3 ==================== @@ -36,6 +36,7 @@ Contributors * Stefan Rijnhart * Ronald Portier * Andrea Stirpe +* Fekete Mihai Maintainer ---------- diff --git a/account_bank_statement_import_mt940_base/__manifest__.py b/account_bank_statement_import_mt940_base/__manifest__.py index 212734b..a88409e 100644 --- a/account_bank_statement_import_mt940_base/__manifest__.py +++ b/account_bank_statement_import_mt940_base/__manifest__.py @@ -1,25 +1,8 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# Copyright (C) 2013-2015 Therp BV -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## +# Copyright (C) 2013-2015 Therp BV + { 'name': 'MT940 Bank Statements Import', - 'version': '10.0.1.0.0', + 'version': '11.0.1.0.0', 'license': 'AGPL-3', 'author': 'Odoo Community Association (OCA), Therp BV', 'website': 'https://github.com/OCA/bank-statement-import', diff --git a/account_bank_statement_import_mt940_base/mt940.py b/account_bank_statement_import_mt940_base/mt940.py index 24a6969..160e8a8 100644 --- a/account_bank_statement_import_mt940_base/mt940.py +++ b/account_bank_statement_import_mt940_base/mt940.py @@ -1,23 +1,7 @@ -# -*- coding: utf-8 -*- +# Copyright (C) 2014-2015 Therp BV . +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). """Generic parser for MT940 files, base for customized versions per bank.""" -############################################################################## -# -# Copyright (C) 2014-2015 Therp BV . -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## + import re import logging from datetime import datetime @@ -138,35 +122,65 @@ class MT940(object): (line[:12], self.mt940_type) ) - def parse(self, data): + def is_mt940_statement(self, line): + """determine if line is the start of a statement""" + if not bool(line.startswith('{4:')): + raise ValueError( + 'The pre processed match %s does not seem to be a' + ' valid %s MT940 format bank statement. Every statement' + ' should start be a dict starting with {4:..' % line + ) + + def pre_process_data(self, data): + matches = [] + self.is_mt940(line=data) + data = data.replace( + '-}', '}').replace('}{', '}\r\n{').replace('\r\n', '\n') + if data.startswith(':940:'): + for statement in data.replace(':940:', '').split(':20:'): + match = '{4:\n:20:' + statement + '}' + matches.append(match) + else: + tag_re = re.compile( + r'(\{4:[^{}]+\})', + re.MULTILINE) + matches = tag_re.findall(data) + return matches + + def parse(self, data, header_lines=None): """Parse mt940 bank statement file contents.""" - self.is_mt940(data) - iterator = data.replace('\r\n', '\n').split('\n').__iter__() - line = None - record_line = '' - try: - while True: - if not self.current_statement: - self.handle_header(line, iterator) - line = iterator.next() - if not self.is_tag(line) and not self.is_footer(line): - record_line = self.add_record_line(line, record_line) - continue + data = data.decode() + matches = self.pre_process_data(data) + for match in matches: + self.is_mt940_statement(line=match) + iterator = '\n'.join( + match.split('\n')[1:-1]).split('\n').__iter__() + line = None + record_line = '' + try: + while True: + if not self.current_statement: + self.handle_header(line, iterator, + header_lines=header_lines) + line = next(iterator) + if not self.is_tag(line) and not self.is_footer(line): + record_line = self.add_record_line(line, record_line) + continue + if record_line: + self.handle_record(record_line) + if self.is_footer(line): + self.handle_footer(line, iterator) + record_line = '' + continue + record_line = line + except StopIteration: + pass + if self.current_statement: if record_line: self.handle_record(record_line) - if self.is_footer(line): - self.handle_footer(line, iterator) record_line = '' - continue - record_line = line - except StopIteration: - pass - if self.current_statement: - if record_line: - self.handle_record(record_line) - record_line = '' - self.statements.append(self.current_statement) - self.current_statement = None + self.statements.append(self.current_statement) + self.current_statement = None return self.currency_code, self.account_number, self.statements def add_record_line(self, line, record_line): @@ -181,10 +195,12 @@ class MT940(object): """determine if a line has a tag""" return line and bool(re.match(self.tag_regex, line)) - def handle_header(self, dummy_line, iterator): + def handle_header(self, dummy_line, iterator, header_lines=None): """skip header lines, create current statement""" - for dummy_i in range(self.header_lines): - iterator.next() + if not header_lines: + header_lines = self.header_lines + for dummy_i in range(header_lines): + next(iterator) self.current_statement = { 'name': None, 'date': None, @@ -229,10 +245,13 @@ class MT940(object): # statement for each 20: tag encountered. if not self.currency_code: self.currency_code = data[7:10] - self.current_statement['balance_start'] = str2amount( - data[0], - data[10:] - ) + self.current_statement['balance_start'] = str2amount( + data[0], + data[10:] + ) + if not self.current_statement['date']: + self.current_statement['date'] = datetime.strptime(data[1:7], + '%y%m%d') def handle_tag_61(self, data): """get transaction values""" @@ -271,7 +290,7 @@ class MT940(object): statement_name = self.current_statement['name'] or '' test_empty_id = re.sub(r'[\s0]', '', statement_name) is_account_number = statement_name.startswith(self.account_number) - if ((not test_empty_id) or is_account_number): + if not test_empty_id or is_account_number: self.current_statement['name'] = '%s-%s' % ( self.account_number, self.current_statement['date'].strftime('%Y-%m-%d'), diff --git a/account_bank_statement_import_mt940_base/test_files/test-sns.940 b/account_bank_statement_import_mt940_base/test_files/test-sns.940 new file mode 100755 index 0000000..a1b032c --- /dev/null +++ b/account_bank_statement_import_mt940_base/test_files/test-sns.940 @@ -0,0 +1,78 @@ +{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4: +:20:0000000000 +:25:NL05SNSB0908244436 +:28C:361/1 +:60F:C171227EUR3026,96 +:61:1712271227D713,13NOVBNL49RABO0166416932 +gerrits glas en schilderwerk +:86:NL49RABO0166416932 gerrits glas en schilderwerk + +Factuur 17227/248/20 + + + +:61:1712271227D10,18NBEA//228ohu/972795 +:86: + +Jumbo Wijchen B.V. >WIJCHEN 27.12.2017 13U38 KV005 4XZJ4Z M +CC:5411 Contactloze betaling + + +:61:1712271227D13,52NINCNL94INGB0000869000 +vitens nv +:86:NL94INGB0000869000 vitens nv + +Europese incasso door:VITENS NV NL-Factuurnr 072304597540 VNKlant +nr 00000000000 BTW 0,77PC 6605 DW 6223, DEC-Incassant ID: NL84ZZZ0 +50695810000-Kenmerk Machtiging: bla + +:61:1712271227D25,61NBEA//229hro/195867 +:86: + +Albert Heijn 1370 >WIJCHEN 27.12.2017 19U13 KV006 70X708 M +CC:5411 + + +:62F:C171227EUR2264,52 +-}{5:} +{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4: +:20:0000000000 +:25:NL05SNSB0908244436 +:28C:362/1 +:60F:C171228EUR2264,52 +:61:1712281228D10,95NINCNL40RABO0127859497 +antagonist b.v. +:86:NL40RABO0127859497 antagonist b.v. + +Europese incasso door:ANTAGONIST B.V. NL-DDWPN954156 AI529942Cweb +share.nl-Incassant ID: NL65ZZZ091364410000-Kenmerk Machtiging: bla + +:61:1712281228C0,00NDIV +:86: + +Mobiel betalen is vanaf nu niet meer mogelijk voor B +ERG M A VAN DEN met Lenovo P2 + + +:62F:C171228EUR2253,57 +-}{5:} +{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4: +:20:0000000000 +:25:NL05SNSB0908244436 +:28C:363/1 +:60F:C171229EUR2253,57 +:61:1712291229D907,29NINCNL19ABNA0427093546 +florius +:86:NL19ABNA0427093546 florius + +Europese incasso door:FLORIUS NL-Verschuldigde bedragen PERIODE 1 +2-2017-Incassant ID: NL42ZZZ080242850000-Kenmerk Machtiging: bla + +:61:1712291229D35,00NINCNL28DEUT0265186439 +stichting derdengelden bucka +:86:NL28DEUT0265186439 stichting derdengelden bucka + +Europese incasso door:STICHTING DERDENGELDEN BUCKAROO-T-Mobile Th +uis B.V.: Rekening dec 2017. Bekijk je rekening op thuismy.t-mobi +le.nl-Incassant ID: NL39ZZZ302317620000-Kenmerk Machtiging: bla +-}{5:} diff --git a/account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 b/account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 new file mode 100644 index 0000000..ae75738 --- /dev/null +++ b/account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 @@ -0,0 +1,11 @@ +:9401: +:20:940S140102 +:25:NL34RABO0142623393 EUR +:28C:0 +:60F:C131231EUR000000004433,52 +:61:140102C000000000400,00N541NONREF +NL66RABO0160878799 +:86:/ORDP//NAME/R. SMITH/ADDR/Green market 74 3311BE Sheepcity Nederl +and NL/REMI/Test money paid by other partner: +/ISDT/2014-01-02 +:62F:C140102EUR000000004833,52 diff --git a/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py b/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py index b25ffc9..9ff19e0 100644 --- a/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py +++ b/account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2017 Onestein () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). @@ -48,13 +47,38 @@ class TestImport(TransactionCase): 'bank_account_id': bank2.id, 'currency_id': self.env.ref('base.EUR').id, }) + bank3 = self.env['res.partner.bank'].create({ + 'acc_number': 'NL05SNSB0908244436', + 'partner_id': self.env.ref('base.main_partner').id, + 'company_id': self.env.ref('base.main_company').id, + 'bank_id': self.env.ref('base.res_bank_1').id, + }) + self.env['account.journal'].create({ + 'name': 'Bank Journal - (test3 mt940)', + 'code': 'TBNK3MT940', + 'type': 'bank', + 'bank_account_id': bank3.id, + 'currency_id': self.env.ref('base.EUR').id, + }) self.data =\ "/BENM//NAME/Cost/REMI/Period 01-10-2013 t/m 31-12-2013/ISDT/20" self.codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', 'ISDT', 'REMI'] + def test_wrong_file_import(self): + """Test wrong file import.""" + testfile = get_module_resource( + 'account_bank_statement_import_mt940_base', + 'test_files', + 'test-wrong-file.940', + ) + parser = MT940() + datafile = open(testfile, 'rb').read() + with self.assertRaises(ValueError): + parser.parse(datafile, header_lines=1) + def test_statement_import(self): - """Test correct creation of single statement.""" + """Test correct creation of single statement ING.""" def _prepare_statement_lines(statements): transact = self.transactions[0] @@ -72,7 +96,7 @@ class TestImport(TransactionCase): ) parser = MT940() datafile = open(testfile, 'rb').read() - statements = parser.parse(datafile) + statements = parser.parse(datafile, header_lines=1) _prepare_statement_lines(statements) @@ -89,8 +113,7 @@ class TestImport(TransactionCase): transact = self.transactions[0] for statement in self.env['account.bank.statement'].browse( - action['context']['statement_ids'] - ): + action['context']['statement_ids']): for line in statement.line_ids: self.assertTrue( line.bank_account_id.acc_number == @@ -121,7 +144,7 @@ class TestImport(TransactionCase): handle_common_subfields(transaction, subfields) def test_statement_import2(self): - """Test correct creation of single statement.""" + """Test correct creation of single statement RABO.""" def _prepare_statement_lines(statements): transact = self.transactions[0] @@ -141,7 +164,7 @@ class TestImport(TransactionCase): parser.header_regex = '^:940:' # Start of header parser.header_lines = 1 # Number of lines to skip datafile = open(testfile, 'rb').read() - statements = parser.parse(datafile) + statements = parser.parse(datafile, header_lines=1) _prepare_statement_lines(statements) @@ -155,11 +178,11 @@ class TestImport(TransactionCase): action = self.env['account.bank.statement.import'].create({ 'data_file': base64.b64encode(datafile), }).import_file() - + # The file contains 4 statements, but only 2 with transactions + self.assertTrue(len(action['context']['statement_ids']) == 2) transact = self.transactions[0] for statement in self.env['account.bank.statement'].browse( - action['context']['statement_ids'] - ): + action['context']['statement_ids']): for line in statement.line_ids: self.assertTrue( line.bank_account_id.acc_number == @@ -168,3 +191,49 @@ class TestImport(TransactionCase): self.assertTrue(line.date) self.assertTrue(line.name == transact['name']) self.assertTrue(line.ref == transact['ref']) + + def test_statement_import3(self): + """Test correct creation of multiple statements SNS.""" + + def _prepare_statement_lines(statements): + transact = self.transactions[0] + for st_vals in statements[2]: + for line_vals in st_vals['transactions']: + line_vals['amount'] = transact['amount'] + line_vals['name'] = transact['name'] + line_vals['account_number'] = transact['account_number'] + line_vals['ref'] = transact['ref'] + + testfile = get_module_resource( + 'account_bank_statement_import_mt940_base', + 'test_files', + 'test-sns.940', + ) + parser = MT940() + datafile = open(testfile, 'rb').read() + statements = parser.parse(datafile, header_lines=1) + + _prepare_statement_lines(statements) + + path_addon = 'odoo.addons.account_bank_statement_import.' + path_file = 'account_bank_statement_import.' + path_class = 'AccountBankStatementImport.' + method = path_addon + path_file + path_class + '_parse_file' + with patch(method) as my_mock: + my_mock.return_value = statements + + action = self.env['account.bank.statement.import'].create({ + 'data_file': base64.b64encode(datafile), + }).import_file() + self.assertTrue(len(action['context']['statement_ids']) == 3) + transact = self.transactions[-1] + for statement in self.env['account.bank.statement'].browse( + action['context']['statement_ids'][-1]): + for line in statement.line_ids: + self.assertTrue( + line.bank_account_id.acc_number == + transact['account_number']) + self.assertTrue(line.amount == transact['amount']) + self.assertTrue(line.date == statement.date) + self.assertTrue(line.name == transact['name']) + self.assertTrue(line.ref == transact['ref'])