OCA
/
bank-statement-import
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Backport of V11 developments (from #137)

pull/125/head
Andrea 6 years ago
parent
f50d296c54
commit
0a18da1c6c
6 changed files with 243 additions and 80 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      account_bank_statement_import_mt940_base/README.rst
  2. 20
      account_bank_statement_import_mt940_base/__manifest__.py
  3. 121
      account_bank_statement_import_mt940_base/mt940.py
  4. 78
      account_bank_statement_import_mt940_base/test_files/test-sns.940
  5. 11
      account_bank_statement_import_mt940_base/test_files/test-wrong-file.940
  6. 88
      account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py

5
account_bank_statement_import_mt940_base/README.rst
View File

@ -1,5 +1,5 @@
.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
:target: http://www.gnu.org/licenses/agpl
.. image:: https://img.shields.io/badge/license-AGPL--3-blue.png
:target: https://www.gnu.org/licenses/agpl
:alt: License: AGPL-3
====================
@ -36,6 +36,7 @@ Contributors
* Stefan Rijnhart <srijnhart@therp.nl>
* Ronald Portier <rportier@therp.nl>
* Andrea Stirpe <a.stirpe@onestein.nl>
* Fekete Mihai <mihai.fekete@forbiom.eu>
Maintainer
----------

20
account_bank_statement_import_mt940_base/__manifest__.py
View File

@ -1,22 +1,6 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (C) 2013-2015 Therp BV <http://therp.nl>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
# Copyright (C) 2013-2015 Therp BV <http://therp.nl>
{
'name': 'MT940 Bank Statements Import',
'version': '10.0.1.0.0',

121
account_bank_statement_import_mt940_base/mt940.py
View File

@ -1,23 +1,7 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2015 Therp BV <http://therp.nl>.
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
"""Generic parser for MT940 files, base for customized versions per bank."""
##############################################################################
#
# Copyright (C) 2014-2015 Therp BV <http://therp.nl>.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import re
import logging
from datetime import datetime
@ -138,35 +122,65 @@ class MT940(object):
(line[:12], self.mt940_type)
)
def parse(self, data):
def is_mt940_statement(self, line):
"""determine if line is the start of a statement"""
if not bool(line.startswith('{4:')):
raise ValueError(
'The pre processed match %s does not seem to be a'
' valid %s MT940 format bank statement. Every statement'
' should start be a dict starting with {4:..' % line
)
def pre_process_data(self, data):
matches = []
self.is_mt940(line=data)
data = data.replace(
'-}', '}').replace('}{', '}\r\n{').replace('\r\n', '\n')
if data.startswith(':940:'):
for statement in data.replace(':940:', '').split(':20:'):
match = '{4:\n:20:' + statement + '}'
matches.append(match)
else:
tag_re = re.compile(
r'(\{4:[^{}]+\})',
re.MULTILINE)
matches = tag_re.findall(data)
return matches
def parse(self, data, header_lines=None):
"""Parse mt940 bank statement file contents."""
self.is_mt940(data)
iterator = data.replace('\r\n', '\n').split('\n').__iter__()
line = None
record_line = ''
try:
while True:
if not self.current_statement:
self.handle_header(line, iterator)
line = iterator.next()
if not self.is_tag(line) and not self.is_footer(line):
record_line = self.add_record_line(line, record_line)
continue
data = data.decode()
matches = self.pre_process_data(data)
for match in matches:
self.is_mt940_statement(line=match)
iterator = '\n'.join(
match.split('\n')[1:-1]).split('\n').__iter__()
line = None
record_line = ''
try:
while True:
if not self.current_statement:
self.handle_header(line, iterator,
header_lines=header_lines)
line = iterator.next()
if not self.is_tag(line) and not self.is_footer(line):
record_line = self.add_record_line(line, record_line)
continue
if record_line:
self.handle_record(record_line)
if self.is_footer(line):
self.handle_footer(line, iterator)
record_line = ''
continue
record_line = line
except StopIteration:
pass
if self.current_statement:
if record_line:
self.handle_record(record_line)
if self.is_footer(line):
self.handle_footer(line, iterator)
record_line = ''
continue
record_line = line
except StopIteration:
pass
if self.current_statement:
if record_line:
self.handle_record(record_line)
record_line = ''
self.statements.append(self.current_statement)
self.current_statement = None
self.statements.append(self.current_statement)
self.current_statement = None
return self.currency_code, self.account_number, self.statements
def add_record_line(self, line, record_line):
@ -181,9 +195,11 @@ class MT940(object):
"""determine if a line has a tag"""
return line and bool(re.match(self.tag_regex, line))
def handle_header(self, dummy_line, iterator):
def handle_header(self, dummy_line, iterator, header_lines=None):
"""skip header lines, create current statement"""
for dummy_i in range(self.header_lines):
if not header_lines:
header_lines = self.header_lines
for dummy_i in range(header_lines):
iterator.next()
self.current_statement = {
'name': None,
@ -229,10 +245,13 @@ class MT940(object):
# statement for each 20: tag encountered.
if not self.currency_code:
self.currency_code = data[7:10]
self.current_statement['balance_start'] = str2amount(
data[0],
data[10:]
)
self.current_statement['balance_start'] = str2amount(
data[0],
data[10:]
)
if not self.current_statement['date']:
self.current_statement['date'] = datetime.strptime(data[1:7],
'%y%m%d')
def handle_tag_61(self, data):
"""get transaction values"""
@ -271,7 +290,7 @@ class MT940(object):
statement_name = self.current_statement['name'] or ''
test_empty_id = re.sub(r'[\s0]', '', statement_name)
is_account_number = statement_name.startswith(self.account_number)
if ((not test_empty_id) or is_account_number):
if not test_empty_id or is_account_number:
self.current_statement['name'] = '%s-%s' % (
self.account_number,
self.current_statement['date'].strftime('%Y-%m-%d'),

78
account_bank_statement_import_mt940_base/test_files/test-sns.940
View File

@ -0,0 +1,78 @@
{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
:20:0000000000
:25:NL05SNSB0908244436
:28C:361/1
:60F:C171227EUR3026,96
:61:1712271227D713,13NOVBNL49RABO0166416932
gerrits glas en schilderwerk
:86:NL49RABO0166416932 gerrits glas en schilderwerk
Factuur 17227/248/20
:61:1712271227D10,18NBEA//228ohu/972795
:86:
Jumbo Wijchen B.V. >WIJCHEN 27.12.2017 13U38 KV005 4XZJ4Z M
CC:5411 Contactloze betaling
:61:1712271227D13,52NINCNL94INGB0000869000
vitens nv
:86:NL94INGB0000869000 vitens nv
Europese incasso door:VITENS NV NL-Factuurnr 072304597540 VNKlant
nr 00000000000 BTW 0,77PC 6605 DW 6223, DEC-Incassant ID: NL84ZZZ0
50695810000-Kenmerk Machtiging: bla
:61:1712271227D25,61NBEA//229hro/195867
:86:
Albert Heijn 1370 >WIJCHEN 27.12.2017 19U13 KV006 70X708 M
CC:5411
:62F:C171227EUR2264,52
-}{5:}
{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
:20:0000000000
:25:NL05SNSB0908244436
:28C:362/1
:60F:C171228EUR2264,52
:61:1712281228D10,95NINCNL40RABO0127859497
antagonist b.v.
:86:NL40RABO0127859497 antagonist b.v.
Europese incasso door:ANTAGONIST B.V. NL-DDWPN954156 AI529942Cweb
share.nl-Incassant ID: NL65ZZZ091364410000-Kenmerk Machtiging: bla
:61:1712281228C0,00NDIV
:86:
Mobiel betalen is vanaf nu niet meer mogelijk voor B
ERG M A VAN DEN met Lenovo P2
:62F:C171228EUR2253,57
-}{5:}
{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
:20:0000000000
:25:NL05SNSB0908244436
:28C:363/1
:60F:C171229EUR2253,57
:61:1712291229D907,29NINCNL19ABNA0427093546
florius
:86:NL19ABNA0427093546 florius
Europese incasso door:FLORIUS NL-Verschuldigde bedragen PERIODE 1
2-2017-Incassant ID: NL42ZZZ080242850000-Kenmerk Machtiging: bla
:61:1712291229D35,00NINCNL28DEUT0265186439
stichting derdengelden bucka
:86:NL28DEUT0265186439 stichting derdengelden bucka
Europese incasso door:STICHTING DERDENGELDEN BUCKAROO-T-Mobile Th
uis B.V.: Rekening dec 2017. Bekijk je rekening op thuismy.t-mobi
le.nl-Incassant ID: NL39ZZZ302317620000-Kenmerk Machtiging: bla
-}{5:}

11
account_bank_statement_import_mt940_base/test_files/test-wrong-file.940
View File

@ -0,0 +1,11 @@
:9401:
:20:940S140102
:25:NL34RABO0142623393 EUR
:28C:0
:60F:C131231EUR000000004433,52
:61:140102C000000000400,00N541NONREF
NL66RABO0160878799
:86:/ORDP//NAME/R. SMITH/ADDR/Green market 74 3311BE Sheepcity Nederl
and NL/REMI/Test money paid by other partner:
/ISDT/2014-01-02
:62F:C140102EUR000000004833,52

88
account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py
View File

@ -48,13 +48,38 @@ class TestImport(TransactionCase):
'bank_account_id': bank2.id,
'currency_id': self.env.ref('base.EUR').id,
})
bank3 = self.env['res.partner.bank'].create({
'acc_number': 'NL05SNSB0908244436',
'partner_id': self.env.ref('base.main_partner').id,
'company_id': self.env.ref('base.main_company').id,
'bank_id': self.env.ref('base.res_bank_1').id,
})
self.env['account.journal'].create({
'name': 'Bank Journal - (test3 mt940)',
'code': 'TBNK3MT940',
'type': 'bank',
'bank_account_id': bank3.id,
'currency_id': self.env.ref('base.EUR').id,
})
self.data =\
"/BENM//NAME/Cost/REMI/Period 01-10-2013 t/m 31-12-2013/ISDT/20"
self.codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', 'ISDT', 'REMI']
def test_wrong_file_import(self):
"""Test wrong file import."""
testfile = get_module_resource(
'account_bank_statement_import_mt940_base',
'test_files',
'test-wrong-file.940',
)
parser = MT940()
datafile = open(testfile, 'rb').read()
with self.assertRaises(ValueError):
parser.parse(datafile, header_lines=1)
def test_statement_import(self):
"""Test correct creation of single statement."""
"""Test correct creation of single statement ING."""
def _prepare_statement_lines(statements):
transact = self.transactions[0]
@ -72,7 +97,7 @@ class TestImport(TransactionCase):
)
parser = MT940()
datafile = open(testfile, 'rb').read()
statements = parser.parse(datafile)
statements = parser.parse(datafile, header_lines=1)
_prepare_statement_lines(statements)
@ -89,8 +114,7 @@ class TestImport(TransactionCase):
transact = self.transactions[0]
for statement in self.env['account.bank.statement'].browse(
action['context']['statement_ids']
):
action['context']['statement_ids']):
for line in statement.line_ids:
self.assertTrue(
line.bank_account_id.acc_number ==
@ -121,7 +145,7 @@ class TestImport(TransactionCase):
handle_common_subfields(transaction, subfields)
def test_statement_import2(self):
"""Test correct creation of single statement."""
"""Test correct creation of single statement RABO."""
def _prepare_statement_lines(statements):
transact = self.transactions[0]
@ -141,7 +165,7 @@ class TestImport(TransactionCase):
parser.header_regex = '^:940:' # Start of header
parser.header_lines = 1 # Number of lines to skip
datafile = open(testfile, 'rb').read()
statements = parser.parse(datafile)
statements = parser.parse(datafile, header_lines=1)
_prepare_statement_lines(statements)
@ -155,11 +179,11 @@ class TestImport(TransactionCase):
action = self.env['account.bank.statement.import'].create({
'data_file': base64.b64encode(datafile),
}).import_file()
# The file contains 4 statements, but only 2 with transactions
self.assertTrue(len(action['context']['statement_ids']) == 2)
transact = self.transactions[0]
for statement in self.env['account.bank.statement'].browse(
action['context']['statement_ids']
):
action['context']['statement_ids']):
for line in statement.line_ids:
self.assertTrue(
line.bank_account_id.acc_number ==
@ -168,3 +192,49 @@ class TestImport(TransactionCase):
self.assertTrue(line.date)
self.assertTrue(line.name == transact['name'])
self.assertTrue(line.ref == transact['ref'])
def test_statement_import3(self):
"""Test correct creation of multiple statements SNS."""
def _prepare_statement_lines(statements):
transact = self.transactions[0]
for st_vals in statements[2]:
for line_vals in st_vals['transactions']:
line_vals['amount'] = transact['amount']
line_vals['name'] = transact['name']
line_vals['account_number'] = transact['account_number']
line_vals['ref'] = transact['ref']
testfile = get_module_resource(
'account_bank_statement_import_mt940_base',
'test_files',
'test-sns.940',
)
parser = MT940()
datafile = open(testfile, 'rb').read()
statements = parser.parse(datafile, header_lines=1)
_prepare_statement_lines(statements)
path_addon = 'odoo.addons.account_bank_statement_import.'
path_file = 'account_bank_statement_import.'
path_class = 'AccountBankStatementImport.'
method = path_addon + path_file + path_class + '_parse_file'
with patch(method) as my_mock:
my_mock.return_value = statements
action = self.env['account.bank.statement.import'].create({
'data_file': base64.b64encode(datafile),
}).import_file()
self.assertTrue(len(action['context']['statement_ids']) == 3)
transact = self.transactions[-1]
for statement in self.env['account.bank.statement'].browse(
action['context']['statement_ids'][-1]):
for line in statement.line_ids:
self.assertTrue(
line.bank_account_id.acc_number ==
transact['account_number'])
self.assertTrue(line.amount == transact['amount'])
self.assertTrue(line.date == statement.date)
self.assertTrue(line.name == transact['name'])
self.assertTrue(line.ref == transact['ref'])
Write Preview
Loading…
Cancel
Save