From d8ed130fa56591f4a735ba2984e53793d1fe6dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Thu, 23 Jul 2015 15:32:38 +0200 Subject: [PATCH] [IMP] rename bank_statement_parse_ to account_bank_statement_import_ Move parserlib to account_bank_statement_import and remove bank_statement_parse module --- account_bank_statement_import_camt/README.rst | 49 ++++ .../__init__.py | 1 + .../__openerp__.py | 34 +++ .../account_bank_statement_import.py | 44 ++++ account_bank_statement_import_camt/camt.py | 238 +++++++++++++++++ .../demo/demo_data.xml | 26 ++ .../test_files/test-camt053.xml | 241 ++++++++++++++++++ .../tests/__init__.py | 23 ++ .../tests/test_import_bank_statement.py | 45 ++++ 9 files changed, 701 insertions(+) create mode 100644 account_bank_statement_import_camt/README.rst create mode 100644 account_bank_statement_import_camt/__init__.py create mode 100644 account_bank_statement_import_camt/__openerp__.py create mode 100644 account_bank_statement_import_camt/account_bank_statement_import.py create mode 100644 account_bank_statement_import_camt/camt.py create mode 100644 account_bank_statement_import_camt/demo/demo_data.xml create mode 100644 account_bank_statement_import_camt/test_files/test-camt053.xml create mode 100644 account_bank_statement_import_camt/tests/__init__.py create mode 100644 account_bank_statement_import_camt/tests/test_import_bank_statement.py diff --git a/account_bank_statement_import_camt/README.rst b/account_bank_statement_import_camt/README.rst new file mode 100644 index 0000000..f027351 --- /dev/null +++ b/account_bank_statement_import_camt/README.rst @@ -0,0 +1,49 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License: AGPL-3 + +Bank Statement Parse Camt +========================= + +Module to import SEPA CAMT.053 Format bank statement files. + +Based on the Banking addons framework. + +Known issues / Roadmap +====================== + +* None + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed feedback +`here `_. + + +Credits +======= + +Contributors +------------ + +* Stefan Rijnhart +* Ronald Portier + +Maintainer +---------- + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit http://odoo-community.org. +This module should make it easy to migrate bank statement import +modules written for earlies versions of Odoo/OpenERP. diff --git a/account_bank_statement_import_camt/__init__.py b/account_bank_statement_import_camt/__init__.py new file mode 100644 index 0000000..7dafcd1 --- /dev/null +++ b/account_bank_statement_import_camt/__init__.py @@ -0,0 +1 @@ +from . import account_bank_statement_import diff --git a/account_bank_statement_import_camt/__openerp__.py b/account_bank_statement_import_camt/__openerp__.py new file mode 100644 index 0000000..dbccdc2 --- /dev/null +++ b/account_bank_statement_import_camt/__openerp__.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (C) 2013-2015 Therp BV +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +{ + 'name': 'CAMT Format Bank Statements Import', + 'version': '0.3', + 'license': 'AGPL-3', + 'author': 'Odoo Community Association (OCA), Therp BV', + 'website': 'https://github.com/OCA/bank-statement-import', + 'category': 'Banking addons', + 'depends': [ + 'account_bank_statement_import', + ], + 'demo': [ + 'demo/demo_data.xml', + ], + 'installable': True, +} diff --git a/account_bank_statement_import_camt/account_bank_statement_import.py b/account_bank_statement_import_camt/account_bank_statement_import.py new file mode 100644 index 0000000..8814670 --- /dev/null +++ b/account_bank_statement_import_camt/account_bank_statement_import.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +"""Add process_camt method to account.bank.statement.import.""" +############################################################################## +# +# Copyright (C) 2013-2015 Therp BV +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +import logging +from openerp import models +from .camt import CamtParser as Parser + + +_logger = logging.getLogger(__name__) + + +class AccountBankStatementImport(models.TransientModel): + """Add process_camt method to account.bank.statement.import.""" + _inherit = 'account.bank.statement.import' + + def _parse_file(self, cr, uid, data_file, context=None): + """Parse a CAMT053 XML file.""" + parser = Parser() + try: + _logger.debug("Try parsing with camt.") + return parser.parse(data_file) + except ValueError: + # Not a camt file, returning super will call next candidate: + _logger.debug("Statement file was not a camt file.", + exc_info=True) + return super(AccountBankStatementImport, self)._parse_file( + cr, uid, data_file, context=context) diff --git a/account_bank_statement_import_camt/camt.py b/account_bank_statement_import_camt/camt.py new file mode 100644 index 0000000..5e6d719 --- /dev/null +++ b/account_bank_statement_import_camt/camt.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- +"""Class to parse camt files.""" +############################################################################## +# +# Copyright (C) 2013-2015 Therp BV +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +import re +from datetime import datetime +from lxml import etree +from openerp.addons.account_bank_statement_import.parserlib import BankStatement + + +class CamtParser(object): + """Parser for camt bank statement import files.""" + + def parse_amount(self, ns, node): + """Parse element that contains Amount and CreditDebitIndicator.""" + if node is None: + return 0.0 + sign = 1 + amount = 0.0 + sign_node = node.xpath('ns:CdtDbtInd', namespaces={'ns': ns}) + if sign_node and sign_node[0].text == 'DBIT': + sign = -1 + amount_node = node.xpath('ns:Amt', namespaces={'ns': ns}) + if amount_node: + amount = sign * float(amount_node[0].text) + return amount + + def add_value_from_node( + self, ns, node, xpath_str, obj, attr_name, join_str=None): + """Add value to object from first or all nodes found with xpath. + + If xpath_str is a list (or iterable), it will be seen as a series + of search path's in order of preference. The first item that results + in a found node will be used to set a value.""" + if not isinstance(xpath_str, (list, tuple)): + xpath_str = [xpath_str] + for search_str in xpath_str: + found_node = node.xpath(search_str, namespaces={'ns': ns}) + if found_node: + if join_str is None: + attr_value = found_node[0].text + else: + attr_value = join_str.join([x.text for x in found_node]) + setattr(obj, attr_name, attr_value) + break + + def parse_transaction_details(self, ns, node, transaction): + """Parse transaction details (message, party, account...).""" + # message + self.add_value_from_node( + ns, node, [ + './ns:RmtInf/ns:Ustrd', + './ns:AddtlTxInf', + './ns:AddtlNtryInf', + ], transaction, 'message') + # eref + self.add_value_from_node( + ns, node, [ + './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref', + './ns:Refs/ns:EndToEndId', + ], + transaction, 'eref' + ) + # remote party values + party_type = 'Dbtr' + party_type_node = node.xpath( + '../../ns:CdtDbtInd', namespaces={'ns': ns}) + if party_type_node and party_type_node[0].text != 'CRDT': + party_type = 'Cdtr' + party_node = node.xpath( + './ns:RltdPties/ns:%s' % party_type, namespaces={'ns': ns}) + if party_node: + self.add_value_from_node( + ns, party_node[0], './ns:Nm', transaction, 'remote_owner') + self.add_value_from_node( + ns, party_node[0], './ns:PstlAdr/ns:Ctry', transaction, + 'remote_owner_country' + ) + address_node = party_node[0].xpath( + './ns:PstlAdr/ns:AdrLine', namespaces={'ns': ns}) + if address_node: + transaction.remote_owner_address = [address_node[0].text] + # Get remote_account from iban or from domestic account: + account_node = node.xpath( + './ns:RltdPties/ns:%sAcct/ns:Id' % party_type, + namespaces={'ns': ns} + ) + if account_node: + iban_node = account_node[0].xpath( + './ns:IBAN', namespaces={'ns': ns}) + if iban_node: + transaction.remote_account = iban_node[0].text + bic_node = node.xpath( + './ns:RltdAgts/ns:%sAgt/ns:FinInstnId/ns:BIC' % party_type, + namespaces={'ns': ns} + ) + if bic_node: + transaction.remote_bank_bic = bic_node[0].text + else: + self.add_value_from_node( + ns, account_node[0], './ns:Othr/ns:Id', transaction, + 'remote_account' + ) + + def parse_transaction(self, ns, node, transaction): + """Parse transaction (entry) node.""" + self.add_value_from_node( + ns, node, './ns:BkTxCd/ns:Prtry/ns:Cd', transaction, + 'transfer_type' + ) + self.add_value_from_node( + ns, node, './ns:BookgDt/ns:Dt', transaction, 'execution_date') + self.add_value_from_node( + ns, node, './ns:ValDt/ns:Dt', transaction, 'value_date') + transaction.transferred_amount = self.parse_amount(ns, node) + details_node = node.xpath( + './ns:NtryDtls/ns:TxDtls', namespaces={'ns': ns}) + if details_node: + self.parse_transaction_details(ns, details_node[0], transaction) + transaction.data = etree.tostring(node) + return transaction + + def get_balance_amounts(self, ns, node): + """Return opening and closing balance. + + Depending on kind of balance and statement, the balance might be in a + different kind of node: + OPBD = OpeningBalance + PRCD = PreviousClosingBalance + ITBD = InterimBalance (first ITBD is start-, second is end-balance) + CLBD = ClosingBalance + """ + start_balance_node = None + end_balance_node = None + for node_name in ['OPBD', 'PRCD', 'CLBD', 'ITBD']: + code_expr = ( + './ns:Bal/ns:Tp/ns:CdOrPrtry/ns:Cd[text()="%s"]/../../..' % + node_name + ) + balance_node = node.xpath(code_expr, namespaces={'ns': ns}) + if balance_node: + if node_name in ['OPBD', 'PRCD']: + start_balance_node = balance_node[0] + elif node_name == 'CLBD': + end_balance_node = balance_node[0] + else: + if not start_balance_node: + start_balance_node = balance_node[0] + if not end_balance_node: + end_balance_node = balance_node[-1] + return ( + self.parse_amount(ns, start_balance_node), + self.parse_amount(ns, end_balance_node) + ) + + def parse_statement(self, ns, node): + """Parse a single Stmt node.""" + statement = BankStatement() + self.add_value_from_node( + ns, node, [ + './ns:Acct/ns:Id/ns:IBAN', + './ns:Acct/ns:Id/ns:Othr/ns:Id', + ], statement, 'local_account' + ) + self.add_value_from_node( + ns, node, './ns:Id', statement, 'statement_id') + self.add_value_from_node( + ns, node, './ns:Acct/ns:Ccy', statement, 'local_currency') + (statement.start_balance, statement.end_balance) = ( + self.get_balance_amounts(ns, node)) + transaction_nodes = node.xpath('./ns:Ntry', namespaces={'ns': ns}) + for entry_node in transaction_nodes: + transaction = statement.create_transaction() + self.parse_transaction(ns, entry_node, transaction) + if statement['transactions']: + statement.date = datetime.strptime( + statement['transactions'][0].execution_date, "%Y-%m-%d") + return statement + + def check_version(self, ns, root): + """Validate validity of camt file.""" + # Check wether it is camt at all: + re_camt = re.compile( + r'(^urn:iso:std:iso:20022:tech:xsd:camt.' + r'|^ISO:camt.)' + ) + if not re_camt.search(ns): + raise ValueError('no camt: ' + ns) + # Check wether version 052 or 053: + re_camt_version = re.compile( + r'(^urn:iso:std:iso:20022:tech:xsd:camt.053.' + r'|^urn:iso:std:iso:20022:tech:xsd:camt.052.' + r'|^ISO:camt.053.' + r'|^ISO:camt.052.)' + ) + if not re_camt_version.search(ns): + raise ValueError('no camt 052 or 053: ' + ns) + # Check GrpHdr element: + root_0_0 = root[0][0].tag[len(ns) + 2:] # strip namespace + if root_0_0 != 'GrpHdr': + raise ValueError('expected GrpHdr, got: ' + root_0_0) + + def parse(self, data): + """Parse a camt.052 or camt.053 file.""" + try: + root = etree.fromstring( + data, parser=etree.XMLParser(recover=True)) + except etree.XMLSyntaxError: + # ABNAmro is known to mix up encodings + root = etree.fromstring( + data.decode('iso-8859-15').encode('utf-8')) + if root is None: + raise ValueError( + 'Not a valid xml file, or not an xml file at all.') + ns = root.tag[1:root.tag.index("}")] + self.check_version(ns, root) + statements = [] + for node in root[0][1:]: + statement = self.parse_statement(ns, node) + if len(statement['transactions']): + statements.append(statement) + return statements diff --git a/account_bank_statement_import_camt/demo/demo_data.xml b/account_bank_statement_import_camt/demo/demo_data.xml new file mode 100644 index 0000000..7fc44d7 --- /dev/null +++ b/account_bank_statement_import_camt/demo/demo_data.xml @@ -0,0 +1,26 @@ + + + + + + Bank Journal - (test camt) + TBNKCAMT + bank + + + + + + + + Your Company + NL77ABNA0574908765 + + + + bank + + + + + diff --git a/account_bank_statement_import_camt/test_files/test-camt053.xml b/account_bank_statement_import_camt/test_files/test-camt053.xml new file mode 100644 index 0000000..1d74d81 --- /dev/null +++ b/account_bank_statement_import_camt/test_files/test-camt053.xml @@ -0,0 +1,241 @@ + + + + TESTBANK/NL/1420561226673 + 2013-01-06T16:20:26.673Z + + + 1234Test/1 + 2 + 2013-01-06T16:20:26.673Z + + 2013-01-05T00:00:00.000Z + 2013-01-05T23:59:59.999Z + + + + NL77ABNA0574908765 + + Example company + + + ABNANL2A + + + + + + + OPBD + + + 15568.27 + CRDT +
+
2013-01-05
+ + + + + + CLBD + + + 15121.12 + CRDT +
+
2013-01-05
+ + + + 754.25 + DBIT + BOOK + +
2013-01-05
+ + +
2013-01-05
+ + + + PMNT + + RDDT + ESDD + + + + EI + + + + + + INNDNL2U20141231000142300002844 + 435005714488-ABNO33052620 + 1880000341866 + + + + 754.25 + + + + + INSURANCE COMPANY TESTX + + TEST STREET 20 + 1234 AB TESTCITY + NL + + + + + NL46ABNA0499998748 + + + + + + + ABNANL2A + + + + + Insurance policy 857239PERIOD 01.01.2013 - 31.12.2013 + + MKB Insurance 859239PERIOD 01.01.2013 - 31.12.2013 + + + + + 594.05 + DBIT + true + BOOK + +
2013-01-05
+ + +
2013-01-05
+ + + + PMNT + + IDDT + UPDD + + + + EIST + + + + + + TESTBANK/NL/20141229/01206408 + TESTBANK/NL/20141229/01206408 + NL22ZZZ524885430000-C0125.1 + + + + 564.05 + + + + + Test Customer + + NL + + + + + NL46ABNA0499998748 + + + + + + + ABNANL2A + + + + + Direct Debit S14 0410 + + + + AC06 + + + Direct debit S14 0410 AC07 Rek.nummer blokkade TESTBANK/NL/20141229/01206408 + + + + + 1405.31 + CRDT + BOOK + +
2013-01-05
+ + +
2013-01-05
+ + + + PMNT + + RCDT + ESCT + + + + ET + + + + + + INNDNL2U20130105000217200000708 + 115 + + + + 1405.31 + + + + + 3rd party Media + + SOMESTREET 570-A + 1276 ML HOUSCITY + NL + + + + + NL69ABNA0522123643 + + + + + + + ABNANL2A + + + + #RD PARTY MEDIA CUSNO 90782 4210773 + + + + + + diff --git a/account_bank_statement_import_camt/tests/__init__.py b/account_bank_statement_import_camt/tests/__init__.py new file mode 100644 index 0000000..cd97bca --- /dev/null +++ b/account_bank_statement_import_camt/tests/__init__.py @@ -0,0 +1,23 @@ +# -*- encoding: utf-8 -*- +"""Test import of bank statement for camt.053.""" +############################################################################## +# +# Copyright (C) 2015 Therp BV . +# +# All other contributions are (C) by their respective contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +from . import test_import_bank_statement diff --git a/account_bank_statement_import_camt/tests/test_import_bank_statement.py b/account_bank_statement_import_camt/tests/test_import_bank_statement.py new file mode 100644 index 0000000..8a1694f --- /dev/null +++ b/account_bank_statement_import_camt/tests/test_import_bank_statement.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +"""Run test to import camt.053 import.""" +############################################################################## +# +# Copyright (C) 2015 Therp BV . +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +from openerp.addons.account_bank_statement_import.tests import ( + TestStatementFile) + + +class TestImport(TestStatementFile): + """Run test to import camt import.""" + + def test_statement_import(self): + """Test correct creation of single statement.""" + transactions = [ + { + 'remote_account': 'NL46ABNA0499998748', + 'transferred_amount': -754.25, + 'value_date': '2013-01-05', + 'ref': '435005714488-ABNO33052620', + }, + ] + # statement name is account number + '-' + date of last 62F line: + self._test_statement_import( + 'account_bank_statement_import_camt', 'test-camt053.xml', + '1234Test/1', + local_account='NL77ABNA0574908765', + start_balance=15568.27, end_balance=15121.12, + transactions=transactions + )