From c4dc160eda0b817d0f64cd647cbdedf64446620d Mon Sep 17 00:00:00 2001 From: ecino Date: Wed, 24 Jan 2018 15:11:26 +0100 Subject: [PATCH] [10.0] Better parse camt groupped transactions (#131) * correctly parse entries with multiple transactions Our bank does it. Care has been taken not to break cases where Ntry/NtryDtls/TxDtls isn't used: if any piece of information isn't found on these nodes, the matching information from the Ntry is used. Most often the entry has a sum or a summary and the transaction details are more precise and specific, so it makes sense to use their contents rather than its. * more complete unit tests for camt parser This checks the whole output data of the parser against its expected output. * add a test case that checks TxDtls decoding * special case for Ntrys without TxDtls * CO-12 - Date Import BVR * FIX tests * CO-1232 Fixed issues with CAMT parser tests --- .../models/parser.py | 70 +++--- .../test_files/golden-camt053-txdtls.pydata | 26 +++ .../test_files/golden-camt053.pydata | 45 ++++ .../test_files/test-camt053-txdtls | 214 ++++++++++++++++++ .../tests/test_import_bank_statement.py | 45 +++- 5 files changed, 368 insertions(+), 32 deletions(-) create mode 100644 account_bank_statement_import_camt/test_files/golden-camt053-txdtls.pydata create mode 100644 account_bank_statement_import_camt/test_files/golden-camt053.pydata create mode 100644 account_bank_statement_import_camt/test_files/test-camt053-txdtls diff --git a/account_bank_statement_import_camt/models/parser.py b/account_bank_statement_import_camt/models/parser.py index 2165774..fedc386 100644 --- a/account_bank_statement_import_camt/models/parser.py +++ b/account_bank_statement_import_camt/models/parser.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Class to parse camt files.""" # © 2013-2016 Therp BV +# Copyright 2017 Open Net Sàrl # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). import re from lxml import etree @@ -46,7 +47,7 @@ class CamtParser(models.AbstractModel): break def parse_transaction_details(self, ns, node, transaction): - """Parse transaction details (message, party, account...).""" + """Parse TxDtls node.""" # message self.add_value_from_node( ns, node, [ @@ -64,9 +65,13 @@ class CamtParser(models.AbstractModel): ns, node, [ './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref', './ns:Refs/ns:EndToEndId', + './ns:Ntry/ns:AcctSvcrRef' ], transaction, 'ref' ) + amount = self.parse_amount(ns, node) + if amount != 0.0: + transaction['amount'] = amount # remote party values party_type = 'Dbtr' party_type_node = node.xpath( @@ -107,10 +112,11 @@ class CamtParser(models.AbstractModel): ns, account_node[0], './ns:Othr/ns:Id', transaction, 'account_number' ) + transaction['data'] = etree.tostring(node) - def parse_transaction(self, ns, node): - """Parse transaction (entry) node.""" - transaction = {} + def parse_entry(self, ns, node): + """Parse an Ntry node and yield transactions""" + transaction = {'name': '/', 'amount': 0} # fallback defaults self.add_value_from_node( ns, node, './ns:BkTxCd/ns:Prtry/ns:Cd', transaction, 'transfer_type' @@ -121,27 +127,30 @@ class CamtParser(models.AbstractModel): ns, node, './ns:BookgDt/ns:Dt', transaction, 'execution_date') self.add_value_from_node( ns, node, './ns:ValDt/ns:Dt', transaction, 'value_date') + amount = self.parse_amount(ns, node) + if amount != 0.0: + transaction['amount'] = amount + self.add_value_from_node( + ns, node, './ns:AddtlNtryInf', transaction, 'name') + self.add_value_from_node( + ns, node, [ + './ns:NtryDtls/ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref', + './ns:NtryDtls/ns:Btch/ns:PmtInfId', + './ns:NtryDtls/ns:TxDtls/ns:Refs/ns:AcctSvcrRef' + ], + transaction, 'ref' + ) - transaction['amount'] = self.parse_amount(ns, node) - - details_node = node.xpath( + details_nodes = node.xpath( './ns:NtryDtls/ns:TxDtls', namespaces={'ns': ns}) - if details_node: - self.parse_transaction_details(ns, details_node[0], transaction) - if not transaction.get('name'): - self.add_value_from_node( - ns, node, './ns:AddtlNtryInf', transaction, 'name') - if not transaction.get('name'): - transaction['name'] = '/' - if not transaction.get('ref'): - self.add_value_from_node( - ns, node, [ - './ns:NtryDtls/ns:Btch/ns:PmtInfId', - ], - transaction, 'ref' - ) - transaction['data'] = etree.tostring(node) - return transaction + if len(details_nodes) == 0: + yield transaction + return + transaction_base = transaction + for node in details_nodes: + transaction = transaction_base.copy() + self.parse_transaction_details(ns, node, transaction) + yield transaction def get_balance_amounts(self, ns, node): """Return opening and closing balance. @@ -193,12 +202,15 @@ class CamtParser(models.AbstractModel): ns, node, './ns:Acct/ns:Ccy', result, 'currency') result['balance_start'], result['balance_end_real'] = ( self.get_balance_amounts(ns, node)) - transaction_nodes = node.xpath('./ns:Ntry', namespaces={'ns': ns}) - result['transactions'] = [] - for entry_node in transaction_nodes: - transaction = self.parse_transaction(ns, entry_node) - if transaction: - result['transactions'].append(transaction) + entry_nodes = node.xpath('./ns:Ntry', namespaces={'ns': ns}) + transactions = [] + for entry_node in entry_nodes: + transactions.extend(self.parse_entry(ns, entry_node)) + result['transactions'] = transactions + result['date'] = sorted(transactions, + key=lambda x: x['date'], + reverse=True + )[0]['date'] return result def check_version(self, ns, root): diff --git a/account_bank_statement_import_camt/test_files/golden-camt053-txdtls.pydata b/account_bank_statement_import_camt/test_files/golden-camt053-txdtls.pydata new file mode 100644 index 0000000..f641bfb --- /dev/null +++ b/account_bank_statement_import_camt/test_files/golden-camt053-txdtls.pydata @@ -0,0 +1,26 @@ +(None, + 'CH1111000000123456789', + [{'balance_end_real': 79443.15, + 'balance_start': 75960.15, + 'date': '2017-03-22', + 'name': '20170323123456789012345', + 'transactions': [{'account_number': 'CH2222000000123456789', + 'amount': 2187.0, + 'data': '\n \n 123456CHCAFEBABE\n \n 01\n 123456CHCAFEBABE\n \n \n 2187.00\n CRDT\n \n \n PMNT\n \n RCDT\n AUTT\n \n \n \n \n \n Banque Cantonale Vaudoise\n \n Place Saint-François\n 14\n 1003\n Lausanne\n CH1\n \n \n \n \n CH2222000000123456789\n \n \n \n \n \n \n POFICHBEXXX\n POSTFINANCE AG\n \n MINGERSTRASSE 20\n 3030 BERNE\n \n \n \n \n \n \n \n \n \n ISR Reference\n \n \n 302388292000011111111111111\n \n ?REJECT?0\n \n \n \n 2017-03-22T20:00:00\n \n \n ', + 'date': '2017-03-22', + 'execution_date': '2017-03-22', + 'name': u'CR\xc9DIT GROUP\xc9 BVR TRAITEMENT DU 22.03.2017 NUM\xc9RO CLIENT 01-70884-3 PAQUET ID: 123456CHCAFEBABE', + 'partner_country': 'CH1', + 'partner_name': 'Banque Cantonale Vaudoise', + 'ref': '302388292000011111111111111', + 'value_date': '2017-03-23'}, + {'account_number': 'CH3333000000123456789', + 'amount': 1296.0, + 'data': '\n \n 123456CHCAFEBABE\n \n 01\n 123456CHCAFEBABE\n \n \n 1296.00\n CRDT\n \n \n PMNT\n \n RCDT\n AUTT\n \n \n \n \n \n Banque Cantonale Vaudoise\n \n Place Saint-François\n 14\n 1003\n Lausanne\n CH2\n \n \n \n \n CH3333000000123456789\n \n \n \n \n \n \n POFICHBEYYY\n POSTFINANCE AG\n \n MINGERSTRASSE 20\n 3030 BERNE\n \n \n \n \n \n \n \n \n \n ISR Reference\n \n \n 302388292000022222222222222\n \n ?REJECT?0\n \n \n \n 2017-03-22T20:00:00\n \n \n ', + 'date': '2017-03-22', + 'execution_date': '2017-03-22', + 'name': u'CR\xc9DIT GROUP\xc9 BVR TRAITEMENT DU 22.03.2017 NUM\xc9RO CLIENT 01-70884-3 PAQUET ID: 123456CHCAFEBABE', + 'partner_country': 'CH2', + 'partner_name': 'Banque Cantonale Vaudoise', + 'ref': '302388292000022222222222222', + 'value_date': '2017-03-23'}]}]) diff --git a/account_bank_statement_import_camt/test_files/golden-camt053.pydata b/account_bank_statement_import_camt/test_files/golden-camt053.pydata new file mode 100644 index 0000000..da29c8d --- /dev/null +++ b/account_bank_statement_import_camt/test_files/golden-camt053.pydata @@ -0,0 +1,45 @@ +(None, + 'NL77ABNA0574908765', + [{'balance_end_real': 15121.12, + 'balance_start': 15568.27, + 'date': '2014-01-05', + 'name': '1234Test/1', + 'transactions': [{'account_bic': 'ABNANL2A', + 'account_number': 'NL46ABNA0499998748', + 'amount': -754.25, + 'data': '\n \n INNDNL2U20141231000142300002844\n 435005714488-ABNO33052620\n 1880000341866\n \n \n \n 754.25\n \n \n \n \n INSURANCE COMPANY TESTX\n \n TEST STREET 20\n 1234 AB TESTCITY\n NL\n \n \n \n \n NL46ABNA0499998748\n \n \n \n \n \n \n ABNANL2A\n \n \n \n \n Insurance policy 857239PERIOD 01.01.2014 - 31.12.2014\n \n MKB Insurance 859239PERIOD 01.01.2014 - 31.12.2014\n \n ', + 'date': '2014-01-05', + 'execution_date': '2014-01-05', + 'name': 'MKB Insurance 859239PERIOD 01.01.2014 - 31.12.2014', + 'note': 'Insurance policy 857239PERIOD 01.01.2014 - 31.12.2014', + 'partner_country': 'NL', + 'partner_name': 'INSURANCE COMPANY TESTX', + 'ref': '435005714488-ABNO33052620', + 'transfer_type': 'EI', + 'value_date': '2014-01-05'}, + {'account_bic': 'ABNANL2A', + 'account_number': 'NL46ABNA0499998748', + 'amount': -594.05, + 'data': '\n \n TESTBANK/NL/20141229/01206408\n TESTBANK/NL/20141229/01206408\n NL22ZZZ524885430000-C0125.1\n \n \n \n 564.05\n \n \n \n \n Test Customer\n \n NL\n \n \n \n \n NL46ABNA0499998748\n \n \n \n \n \n \n ABNANL2A\n \n \n \n \n Direct Debit S14 0410\n \n \n \n AC06\n \n \n Direct debit S14 0410 AC07 Rek.nummer blokkade TESTBANK/NL/20141229/01206408\n \n ', + 'date': '2014-01-05', + 'execution_date': '2014-01-05', + 'name': 'Direct debit S14 0410 AC07 Rek.nummer blokkade TESTBANK/NL/20141229/01206408', + 'note': 'Direct Debit S14 0410', + 'partner_country': 'NL', + 'partner_name': 'Test Customer', + 'ref': 'TESTBANK/NL/20141229/01206408', + 'transfer_type': 'EIST', + 'value_date': '2014-01-05'}, + {'account_bic': 'ABNANL2A', + 'account_number': 'NL69ABNA0522123643', + 'amount': 1405.31, + 'data': '\n \n INNDNL2U20140105000217200000708\n 115\n \n \n \n 1405.31\n \n \n \n \n 3rd party Media\n \n SOMESTREET 570-A\n 1276 ML HOUSCITY\n NL\n \n \n \n \n NL69ABNA0522123643\n \n \n \n \n \n \n ABNANL2A\n \n \n \n #RD PARTY MEDIA CUSNO 90782 4210773\n \n ', + 'date': '2014-01-05', + 'execution_date': '2014-01-05', + 'name': '#RD PARTY MEDIA CUSNO 90782 4210773', + 'note': 'INNDNL2U20140105000217200000708', + 'partner_country': 'NL', + 'partner_name': '3rd party Media', + 'ref': '115', + 'transfer_type': 'ET', + 'value_date': '2014-01-05'}]}]) diff --git a/account_bank_statement_import_camt/test_files/test-camt053-txdtls b/account_bank_statement_import_camt/test_files/test-camt053-txdtls new file mode 100644 index 0000000..cf7c38f --- /dev/null +++ b/account_bank_statement_import_camt/test_files/test-camt053-txdtls @@ -0,0 +1,214 @@ + + + + + 20170323312345678900000 + 2017-03-23T14:47:00 + + 1 + true + + Test + + + 20170323123456789012345 + 58 + 2017-03-23T14:47:00 + + 2017-03-23T00:00:00 + 2017-03-23T23:59:59 + + + + CH1111000000123456789 + + + Open Net S. à r.l. Prilly + + + + + + OPBD + + + 75960.15 + CRDT +
+
2017-03-22
+ + + + + + CLBD + + + 79443.15 + CRDT +
+
2017-03-23
+ + + + 012345678 + 3483.00 + CRDT + false + BOOK + +
2017-03-22
+ + +
2017-03-23
+ + 20170323001234567891234567891234 + + + PMNT + + RCDT + VCOM + + + + + + 2 + + + + 123456CHCAFEBABE + + 01 + 123456CHCAFEBABE + + + 2187.00 + CRDT + + + PMNT + + RCDT + AUTT + + + + + + Banque Cantonale Vaudoise + + Place Saint-François + 14 + 1003 + Lausanne + CH1 + + + + + CH2222000000123456789 + + + + + + + POFICHBEXXX + POSTFINANCE AG + + MINGERSTRASSE 20 + 3030 BERNE + + + + + + + + + + ISR Reference + + + 302388292000011111111111111 + + ?REJECT?0 + + + + 2017-03-22T20:00:00 + + + + + 123456CHCAFEBABE + + 01 + 123456CHCAFEBABE + + + 1296.00 + CRDT + + + PMNT + + RCDT + AUTT + + + + + + Banque Cantonale Vaudoise + + Place Saint-François + 14 + 1003 + Lausanne + CH2 + + + + + CH3333000000123456789 + + + + + + + POFICHBEYYY + POSTFINANCE AG + + MINGERSTRASSE 20 + 3030 BERNE + + + + + + + + + + ISR Reference + + + 302388292000022222222222222 + + ?REJECT?0 + + + + 2017-03-22T20:00:00 + + + + CRÉDIT GROUPÉ BVR TRAITEMENT DU 22.03.2017 NUMÉRO CLIENT 01-70884-3 PAQUET ID: 123456CHCAFEBABE + + + + diff --git a/account_bank_statement_import_camt/tests/test_import_bank_statement.py b/account_bank_statement_import_camt/tests/test_import_bank_statement.py index b3c57aa..840f005 100644 --- a/account_bank_statement_import_camt/tests/test_import_bank_statement.py +++ b/account_bank_statement_import_camt/tests/test_import_bank_statement.py @@ -1,12 +1,53 @@ # -*- coding: utf-8 -*- """Run test to import camt.053 import.""" # © 2013-2016 Therp BV +# Copyright 2017 Open Net Sàrl # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). import base64 +import difflib +import pprint +import tempfile + from odoo.tests.common import TransactionCase from odoo.tools.misc import file_open +DATA_DIR = 'account_bank_statement_import_camt/test_files/' + + +class TestParser(TransactionCase): + """Tests for the camt parser itself.""" + def setUp(self): + super(TestParser, self).setUp() + self.parser = self.env['account.bank.statement.import.camt.parser'] + + def _do_parse_test(self, inputfile, goldenfile): + with file_open(inputfile) as testfile: + data = testfile.read() + res = self.parser.parse(data) + with tempfile.NamedTemporaryFile(suffix='.pydata') as temp: + pprint.pprint(res, temp) + with file_open(goldenfile) as golden: + temp.seek(0) + diff = list( + difflib.unified_diff(golden.readlines(), temp.readlines(), + golden.name, temp.name)) + if len(diff) > 2: + self.fail( + "actual output doesn't match exptected output:\n%s" % + "".join(diff)) + + def test_parse(self): + self._do_parse_test( + DATA_DIR + 'test-camt053', + DATA_DIR + 'golden-camt053.pydata') + + def test_parse_txdtls(self): + self._do_parse_test( + DATA_DIR + 'test-camt053-txdtls', + DATA_DIR + 'golden-camt053-txdtls.pydata') + + class TestImport(TransactionCase): """Run test to import camt import.""" transactions = [ @@ -36,9 +77,7 @@ class TestImport(TransactionCase): def test_statement_import(self): """Test correct creation of single statement.""" action = {} - with file_open( - 'account_bank_statement_import_camt/test_files/test-camt053' - ) as testfile: + with file_open(DATA_DIR + 'test-camt053') as testfile: action = self.env['account.bank.statement.import'].create({ 'data_file': base64.b64encode(testfile.read()), }).import_file()