[10.0][MIG] account_bank_statement_import_mt940_base: port to V10

account_bank_statement_import_mt940_base/README.rst

@@ -1,6 +1,8 @@
 .. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+   :target: http://www.gnu.org/licenses/agpl
    :alt: License: AGPL-3
 
+====================
 Bank Statement MT940
 ====================
 
@@ -10,7 +12,7 @@ interpretation of it, this addon alone won't help you much. It is rather
 intended to be used by other addons to implement the dialect specific to a
 certain bank.
 
-See bank_statement_parse_nl_ing_mt940 for an example on how to use it.
+See account_bank_statement_import_mt940_nl_ing for an example on how to use it.
 
 Known issues / Roadmap
 ======================
@@ -20,11 +22,10 @@ Known issues / Roadmap
 Bug Tracker
 ===========
 
-Bugs are tracked on `GitHub Issues <https://github.com/OCA/bank-statement-import/issues>`_.
-In case of trouble, please check there if your issue has already been reported.
-If you spotted it first, help us smashing it by providing a detailed and welcomed feedback
-`here <https://github.com/OCA/bank-statement-import/issues/new?body=module:%20account_bank_statement_import%0Aversion:%208.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
-
+Bugs are tracked on `GitHub Issues
+<https://github.com/OCA/bank-statement-import/issues>`_. In case of trouble, please
+check there if your issue has already been reported. If you spotted it first,
+help us smash it by providing detailed and welcomed feedback.
 
 Credits
 =======
@@ -34,10 +35,12 @@ Contributors
 
 * Stefan Rijnhart <srijnhart@therp.nl>
 * Ronald Portier <rportier@therp.nl>
+* Andrea Stirpe <a.stirpe@onestein.nl>
 
 Maintainer
 ----------
 
+
 .. image:: https://odoo-community.org/logo.png
    :alt: Odoo Community Association
    :target: https://odoo-community.org
@@ -48,6 +51,4 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose
 mission is to support the collaborative development of Odoo features and
 promote its widespread use.
 
-To contribute to this module, please visit http://odoo-community.org.
-This module should make it easy to migrate bank statement import
-modules written for earlies versions of Odoo/OpenERP.
+To contribute to this module, please visit https://odoo-community.org.

account_bank_statement_import_mt940_base/__manifest__.py

@@ -19,7 +19,7 @@
 ##############################################################################
 {
     'name': 'MT940 Bank Statements Import',
-    'version': '8.0.1.1.1',
+    'version': '10.0.1.0.0',
     'license': 'AGPL-3',
     'author': 'Odoo Community Association (OCA), Therp BV',
     'website': 'https://github.com/OCA/bank-statement-import',
@@ -27,5 +27,5 @@
     'depends': [
         'account_bank_statement_import',
     ],
-    'installable': False
+    'installable': True
 }

account_bank_statement_import_mt940_base/mt940.py

@@ -22,9 +22,6 @@ import re
 import logging
 from datetime import datetime
 
-from openerp.addons.account_bank_statement_import.parserlib import (
-    BankStatement)
-
 
 def str2amount(sign, amount_str):
     """Convert sign (C or D) and amount in string to signed amount (float)."""
@@ -68,13 +65,11 @@ def get_counterpart(transaction, subfield):
     if not subfield:
         return  # subfield is empty
     if len(subfield) >= 1 and subfield[0]:
-        transaction.remote_account = subfield[0]
+        transaction.update({'account_number': subfield[0]})
     if len(subfield) >= 2 and subfield[1]:
-        transaction.remote_bank_bic = subfield[1]
+        transaction.update({'account_bic': subfield[1]})
     if len(subfield) >= 3 and subfield[2]:
-        transaction.remote_owner = subfield[2]
-    if len(subfield) >= 4 and subfield[3]:
-        transaction.remote_owner_city = subfield[3]
+        transaction.update({'partner_name': subfield[2]})
 
 
 def handle_common_subfields(transaction, subfields):
@@ -83,11 +78,11 @@ def handle_common_subfields(transaction, subfields):
     for counterpart_field in ['CNTP', 'BENM', 'ORDP']:
         if counterpart_field in subfields:
             get_counterpart(transaction, subfields[counterpart_field])
-    if not transaction.message:
-        transaction.message = ''
+    if not transaction.get('name'):
+        transaction['name'] = ''
     # REMI: Remitter information (text entered by other party on trans.):
     if 'REMI' in subfields:
-        transaction.message += (
+        transaction['name'] += (
             subfields['REMI'][2]
             # this might look like
             # /REMI/USTD//<remittance info>/
@@ -101,10 +96,10 @@ def handle_common_subfields(transaction, subfields):
         )
     # EREF: End-to-end reference
     if 'EREF' in subfields:
-        transaction.message += '/'.join(filter(bool, subfields['EREF']))
+        transaction['name'] += '/'.join(filter(bool, subfields['EREF']))
     # Get transaction reference subfield (might vary):
-    if transaction.eref in subfields:
-        transaction.eref = ''.join(subfields[transaction.eref])
+    if transaction.get('ref') in subfields:
+        transaction['ref'] = ''.join(subfields[transaction['ref']])
 
 
 class MT940(object):
@@ -131,6 +126,8 @@ class MT940(object):
         self.current_statement = None
         self.current_transaction = None
         self.statements = []
+        self.currency_code = None
+        self.account_number = None
 
     def is_mt940(self, line):
         """determine if a line is the header of a statement"""
@@ -170,7 +167,7 @@ class MT940(object):
                 record_line = ''
             self.statements.append(self.current_statement)
             self.current_statement = None
-        return self.statements
+        return self.currency_code, self.account_number, self.statements
 
     def add_record_line(self, line, record_line):
         record_line += line
@@ -188,7 +185,13 @@ class MT940(object):
         """skip header lines, create current statement"""
         for dummy_i in range(self.header_lines):
             iterator.next()
-        self.current_statement = BankStatement()
+        self.current_statement = {
+            'name': None,
+            'date': None,
+            'balance_start': 0.0,
+            'balance_end_real': 0.0,
+            'transactions': []
+        }
 
     def handle_footer(self, dummy_line, dummy_iterator):
         """add current statement to list, reset state"""
@@ -199,7 +202,7 @@ class MT940(object):
         """find a function to handle the record represented by line"""
         tag_match = re.match(self.tag_regex, line)
         tag = tag_match.group(0).strip(':')
-        if not hasattr(self, 'handle_tag_%s' % tag):
+        if not hasattr(self, 'handle_tag_%s' % tag):  # pragma: no cover
             logging.error('Unknown tag %s', tag)
             logging.error(line)
             return
@@ -213,7 +216,7 @@ class MT940(object):
     def handle_tag_25(self, data):
         """Handle tag 25: local bank account information."""
         data = data.replace('EUR', '').replace('.', '').strip()
-        self.current_statement.local_account = data
+        self.account_number = data
 
     def handle_tag_28C(self, data):
         """Sequence number within batch - normally only zeroes."""
@@ -224,18 +227,21 @@ class MT940(object):
         # For the moment only first 60F record
         # The alternative would be to split the file and start a new
         # statement for each 20: tag encountered.
-        stmt = self.current_statement
-        if not stmt.local_currency:
-            stmt.local_currency = data[7:10]
-            stmt.start_balance = str2amount(data[0], data[10:])
+        if not self.currency_code:
+            self.currency_code = data[7:10]
+            self.current_statement['balance_start'] = str2amount(
+                data[0],
+                data[10:]
+            )
 
     def handle_tag_61(self, data):
         """get transaction values"""
-        transaction = self.current_statement.create_transaction()
-        self.current_transaction = transaction
-        transaction.execution_date = datetime.strptime(data[:6], '%y%m%d')
-        transaction.value_date = datetime.strptime(data[:6], '%y%m%d')
-        #  ...and the rest already is highly bank dependent
+        self.current_statement['transactions'].append({})
+        self.current_transaction = self.current_statement['transactions'][-1]
+        self.current_transaction['date'] = datetime.strptime(
+            data[:6],
+            '%y%m%d'
+        )
 
     def handle_tag_62F(self, data):
         """Get ending balance, statement date and id.
@@ -251,19 +257,24 @@ class MT940(object):
         Depending on the bank, there might be multiple 62F tags in the import
         file. The last one counts.
         """
-        stmt = self.current_statement
-        stmt.end_balance = str2amount(data[0], data[10:])
-        stmt.date = datetime.strptime(data[1:7], '%y%m%d')
+
+        self.current_statement['balance_end_real'] = str2amount(
+            data[0],
+            data[10:]
+        )
+        self.current_statement['date'] = datetime.strptime(data[1:7], '%y%m%d')
+
         # Only replace logically empty (only whitespace or zeroes) id's:
         # But do replace statement_id's added before (therefore starting
         # with local_account), because we need the date on the last 62F
         # record.
-        test_empty_id = re.sub(r'[\s0]', '', stmt.statement_id)
-        if ((not test_empty_id) or
-                (stmt.statement_id.startswith(stmt.local_account))):
-            stmt.statement_id = '%s-%s' % (
-                stmt.local_account,
-                stmt.date.strftime('%Y-%m-%d'),
+        statement_name = self.current_statement['name'] or ''
+        test_empty_id = re.sub(r'[\s0]', '', statement_name)
+        is_account_number = statement_name.startswith(self.account_number)
+        if ((not test_empty_id) or is_account_number):
+            self.current_statement['name'] = '%s-%s' % (
+                self.account_number,
+                self.current_statement['date'].strftime('%Y-%m-%d'),
             )
 
     def handle_tag_64(self, data):

account_bank_statement_import_mt940_base/test_files/test-ing.940

@@ -0,0 +1,61 @@
+{1:F01INGBNL2ABXXX0000000000}
+{2:I940INGBNL2AXXXN}
+{4:
+:20:P140220000000001
+:25:NL77INGB0574908765EUR
+:28C:0000
+0
+:60F:C140219EUR662,23
+:61:1402200220C1,56NTRFEREF//00000000001
+005
+/TRCD/00100/
+:86:/EREF/EV12341REP1231456T1234//CNTP/NL32INGB0000012345/INGBNL2
+A/ING BANK NV INZAKE WEB///REMI/USTD//EV10001REP1000000T1000/
+:61:1402200220D1,57NTRFPREF//00000000001006
+/TRCD/00200/
+:86:/PREF/M000000003333333//REMI/USTD//TOTAAL 1 VZ/
+:61:1402200220C1,57NRTIEREF//00000000001007
+/TRCD/00190/
+:86:/RTRN/MS03//EREF/20120123456789//CNTP/NL32INGB0000012345/INGB
+NL2A/J.Janssen///REMI/USTD//Factuurnr 123456 Klantnr 00123/
+:61:1402200220D1,14NDDTEREF//00000000001009
+/TRCD/010
+16
+/
+:86:/EREF/EV123R
+EP123412T1234//MARF/MND
+-
+EV01//CSID/NL32ZZZ9999999
+91234//CNTP/NL32INGB0000012345/INGBNL2A/ING Bank N.V. inzake WeB/
+//REMI/USTD//EV123REP123412T1234/
+:61:1402200220C1,45NDDTPREF//00000000001008
+/TRCD/01000/
+:86:/PREF/M000000001111111/
+/CSID/
+NL32ZZZ999999991234
+/
+/REMI/USTD//
+TOTAAL 1 POSTEN/
+:61:1402200220D12,75NRTIEREF//00000000001010
+/TRCD/01315/
+:86:/RTRN/MS03//EREF/20120501P0123478//MARF/MND
+-
+120123//CSID/NL32
+ZZZ999999991234//CNTP/NL32INGB0000012345/INGBNL2A/J.Janssen///REM
+I/USTD//CO
+NTRIBUTIE FEB 2014/
+:61:1402200220C32,00NTRF9001123412341234//00000000001011
+/TRCD/00108/
+:86:/EREF/15814016000676480//CNTP/NL32INGB0000012345/INGBNL2A/J.J
+anssen///REMI/STRD/CUR/9001123412341234/
+:61:1402200220D119,00NTRF1070123412341234//00000000001012
+/
+TRCD/00108/
+:86:/EREF/15614016000384600//CNTP/NL32INGB0000012345/INGBNL2A/ING
+BANK NV///REMI/STRD/CUR/1070123412341234/
+:62F:C140220EUR564,35
+:64:C140220EUR564,35
+:65:C140221EUR564,35
+:65:C140224EUR564,35
+:86:/SUM/4/4/134,46/36,58/
+-}

account_bank_statement_import_mt940_base/test_files/test-rabo.swi

@@ -0,0 +1,29 @@
+:940:
+:20:940S140102
+:25:NL34RABO0142623393 EUR
+:28C:0
+:60F:C131231EUR000000004433,52
+:61:140102C000000000400,00N541NONREF
+NL66RABO0160878799
+:86:/ORDP//NAME/R. SMITH/ADDR/Green market 74 3311BE Sheepcity Nederl
+and NL/REMI/Test money paid by other partner:
+/ISDT/2014-01-02
+:62F:C140102EUR000000004833,52
+:20:940S140103
+:25:NL34RABO0142623393 EUR
+:28C:0
+:60F:C140102EUR000000004833,52
+:62F:C140103EUR000000004833,52
+:20:940S140106
+:25:NL34RABO0142623393 EUR
+:28C:0
+:60F:C140103EUR000000004833,52
+:61:140101D000000000034,61N093NONREF
+:86:/BENM//NAME/Kosten/REMI/Periode 01-10-2013 t/m 31-12-2013/ISDT/20
+14-01-01
+:62F:C140106EUR000000004798,91
+:20:940S140107
+:25:NL34RABO0142623393 EUR
+:28C:0
+:60F:C140106EUR000000004798,91
+:62F:C140107EUR000000004798,91

account_bank_statement_import_mt940_base/tests/__init__.py

@@ -0,0 +1 @@
+from . import test_import_bank_statement

account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py

@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Onestein (<http://www.onestein.eu>)
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+
+import base64
+from mock import patch
+from odoo.tests.common import TransactionCase
+from odoo.modules.module import get_module_resource
+from ..mt940 import MT940, get_subfields, handle_common_subfields
+
+
+class TestImport(TransactionCase):
+    """Run test to import mt940 import."""
+    transactions = [
+        {
+            'account_number': 'NL46ABNA0499998748',
+            'amount': -754.25,
+            'ref': '435005714488-ABNO33052620',
+            'name': 'test line',
+        },
+    ]
+
+    def setUp(self):
+        super(TestImport, self).setUp()
+        bank1 = self.env['res.partner.bank'].create({
+            'acc_number': 'NL77INGB0574908765',
+            'partner_id': self.env.ref('base.main_partner').id,
+            'company_id': self.env.ref('base.main_company').id,
+            'bank_id': self.env.ref('base.res_bank_1').id,
+        })
+        self.env['account.journal'].create({
+            'name': 'Bank Journal - (test1 mt940)',
+            'code': 'TBNK1MT940',
+            'type': 'bank',
+            'bank_account_id': bank1.id,
+            'currency_id': self.env.ref('base.EUR').id,
+        })
+        bank2 = self.env['res.partner.bank'].create({
+            'acc_number': 'NL34RABO0142623393',
+            'partner_id': self.env.ref('base.main_partner').id,
+            'company_id': self.env.ref('base.main_company').id,
+            'bank_id': self.env.ref('base.res_bank_1').id,
+        })
+        self.env['account.journal'].create({
+            'name': 'Bank Journal - (test2 mt940)',
+            'code': 'TBNK2MT940',
+            'type': 'bank',
+            'bank_account_id': bank2.id,
+            'currency_id': self.env.ref('base.EUR').id,
+        })
+
+        self.data =\
+            "/BENM//NAME/Cost/REMI/Period 01-10-2013 t/m 31-12-2013/ISDT/20"
+        self.codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', 'ISDT', 'REMI']
+
+    def test_statement_import(self):
+        """Test correct creation of single statement."""
+
+        def _prepare_statement_lines(statements):
+            transact = self.transactions[0]
+            for st_vals in statements[2]:
+                for line_vals in st_vals['transactions']:
+                    line_vals['amount'] = transact['amount']
+                    line_vals['name'] = transact['name']
+                    line_vals['account_number'] = transact['account_number']
+                    line_vals['ref'] = transact['ref']
+
+        testfile = get_module_resource(
+            'account_bank_statement_import_mt940_base',
+            'test_files',
+            'test-ing.940',
+        )
+        parser = MT940()
+        datafile = open(testfile, 'rb').read()
+        statements = parser.parse(datafile)
+
+        _prepare_statement_lines(statements)
+
+        path_addon = 'odoo.addons.account_bank_statement_import.'
+        path_file = 'account_bank_statement_import.'
+        path_class = 'AccountBankStatementImport.'
+        method = path_addon + path_file + path_class + '_parse_file'
+        with patch(method) as my_mock:
+            my_mock.return_value = statements
+
+            action = self.env['account.bank.statement.import'].create({
+                'data_file': base64.b64encode(datafile),
+            }).import_file()
+
+            transact = self.transactions[0]
+            for statement in self.env['account.bank.statement'].browse(
+                action['context']['statement_ids']
+            ):
+                for line in statement.line_ids:
+                    self.assertTrue(
+                        line.bank_account_id.acc_number ==
+                        transact['account_number'])
+                    self.assertTrue(line.amount == transact['amount'])
+                    self.assertTrue(line.date == '2014-02-20')
+                    self.assertTrue(line.name == transact['name'])
+                    self.assertTrue(line.ref == transact['ref'])
+
+    def test_get_subfields(self):
+        """Unit Test function get_subfields()."""
+
+        res = get_subfields(self.data, self.codewords)
+        espected_res = {
+            'BENM': [''],
+            'NAME': ['Cost'],
+            'REMI': ['Period 01-10-2013 t', 'm 31-12-2013'],
+            'ISDT': ['20'],
+        }
+        self.assertTrue(res == espected_res)
+
+    def test_handle_common_subfields(self):
+        """Unit Test function handle_common_subfields()."""
+
+        subfields = get_subfields(self.data, self.codewords)
+        transaction = self.transactions[0]
+
+        handle_common_subfields(transaction, subfields)
+
+    def test_statement_import2(self):
+        """Test correct creation of single statement."""
+
+        def _prepare_statement_lines(statements):
+            transact = self.transactions[0]
+            for st_vals in statements[2]:
+                for line_vals in st_vals['transactions']:
+                    line_vals['amount'] = transact['amount']
+                    line_vals['name'] = transact['name']
+                    line_vals['account_number'] = transact['account_number']
+                    line_vals['ref'] = transact['ref']
+
+        testfile = get_module_resource(
+            'account_bank_statement_import_mt940_base',
+            'test_files',
+            'test-rabo.swi',
+        )
+        parser = MT940()
+        parser.header_regex = '^:940:'  # Start of header
+        parser.header_lines = 1  # Number of lines to skip
+        datafile = open(testfile, 'rb').read()
+        statements = parser.parse(datafile)
+
+        _prepare_statement_lines(statements)
+
+        path_addon = 'odoo.addons.account_bank_statement_import.'
+        path_file = 'account_bank_statement_import.'
+        path_class = 'AccountBankStatementImport.'
+        method = path_addon + path_file + path_class + '_parse_file'
+        with patch(method) as my_mock:
+            my_mock.return_value = statements
+
+            action = self.env['account.bank.statement.import'].create({
+                'data_file': base64.b64encode(datafile),
+            }).import_file()
+
+            transact = self.transactions[0]
+            for statement in self.env['account.bank.statement'].browse(
+                action['context']['statement_ids']
+            ):
+                for line in statement.line_ids:
+                    self.assertTrue(
+                        line.bank_account_id.acc_number ==
+                        transact['account_number'])
+                    self.assertTrue(line.amount == transact['amount'])
+                    self.assertTrue(line.date)
+                    self.assertTrue(line.name == transact['name'])
+                    self.assertTrue(line.ref == transact['ref'])