From 261f3a0213b972abbe1da6c83de1851ad8122163 Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Thu, 10 Nov 2016 17:18:30 +0100 Subject: [PATCH] migrate account_bank_statement_import_camt --- account_bank_statement_import_camt/README.rst | 1 + .../__init__.py | 6 +- .../__manifest__.py | 26 ++--- .../account_bank_statement_import.py | 44 -------- account_bank_statement_import_camt/camt.py | 100 +++++++++--------- .../demo/demo_data.xml | 39 +++---- .../models/__init__.py | 4 + .../models/account_bank_statement_import.py | 42 ++++++++ .../{test-camt053.xml => test-camt053} | 0 .../tests/__init__.py | 22 +--- .../tests/test_import_bank_statement.py | 84 ++++++++------- .../views/account_bank_statement_import.xml | 15 +++ 12 files changed, 182 insertions(+), 201 deletions(-) delete mode 100644 account_bank_statement_import_camt/account_bank_statement_import.py create mode 100644 account_bank_statement_import_camt/models/__init__.py create mode 100644 account_bank_statement_import_camt/models/account_bank_statement_import.py rename account_bank_statement_import_camt/test_files/{test-camt053.xml => test-camt053} (100%) create mode 100644 account_bank_statement_import_camt/views/account_bank_statement_import.xml diff --git a/account_bank_statement_import_camt/README.rst b/account_bank_statement_import_camt/README.rst index f027351..a564f7a 100644 --- a/account_bank_statement_import_camt/README.rst +++ b/account_bank_statement_import_camt/README.rst @@ -28,6 +28,7 @@ Credits Contributors ------------ +* Holger Brunn * Stefan Rijnhart * Ronald Portier diff --git a/account_bank_statement_import_camt/__init__.py b/account_bank_statement_import_camt/__init__.py index 7dafcd1..6350c71 100644 --- a/account_bank_statement_import_camt/__init__.py +++ b/account_bank_statement_import_camt/__init__.py @@ -1 +1,5 @@ -from . import account_bank_statement_import +# -*- coding: utf-8 -*- +# © 2013-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from . import models +from . import camt diff --git a/account_bank_statement_import_camt/__manifest__.py b/account_bank_statement_import_camt/__manifest__.py index 86ce0c5..92e10bd 100644 --- a/account_bank_statement_import_camt/__manifest__.py +++ b/account_bank_statement_import_camt/__manifest__.py @@ -1,25 +1,9 @@ # -*- coding: utf-8 -*- -############################################################################## -# -# Copyright (C) 2013-2015 Therp BV -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## +# © 2013-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). { 'name': 'CAMT Format Bank Statements Import', - 'version': '8.0.0.4.0', + 'version': '9.0.1.0.0', 'license': 'AGPL-3', 'author': 'Odoo Community Association (OCA), Therp BV', 'website': 'https://github.com/OCA/bank-statement-import', @@ -30,5 +14,7 @@ 'demo': [ 'demo/demo_data.xml', ], - 'installable': False, + 'data': [ + 'views/account_bank_statement_import.xml', + ], } diff --git a/account_bank_statement_import_camt/account_bank_statement_import.py b/account_bank_statement_import_camt/account_bank_statement_import.py deleted file mode 100644 index 8814670..0000000 --- a/account_bank_statement_import_camt/account_bank_statement_import.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -"""Add process_camt method to account.bank.statement.import.""" -############################################################################## -# -# Copyright (C) 2013-2015 Therp BV -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## -import logging -from openerp import models -from .camt import CamtParser as Parser - - -_logger = logging.getLogger(__name__) - - -class AccountBankStatementImport(models.TransientModel): - """Add process_camt method to account.bank.statement.import.""" - _inherit = 'account.bank.statement.import' - - def _parse_file(self, cr, uid, data_file, context=None): - """Parse a CAMT053 XML file.""" - parser = Parser() - try: - _logger.debug("Try parsing with camt.") - return parser.parse(data_file) - except ValueError: - # Not a camt file, returning super will call next candidate: - _logger.debug("Statement file was not a camt file.", - exc_info=True) - return super(AccountBankStatementImport, self)._parse_file( - cr, uid, data_file, context=context) diff --git a/account_bank_statement_import_camt/camt.py b/account_bank_statement_import_camt/camt.py index aed01db..d5fa82d 100644 --- a/account_bank_statement_import_camt/camt.py +++ b/account_bank_statement_import_camt/camt.py @@ -1,28 +1,9 @@ # -*- coding: utf-8 -*- """Class to parse camt files.""" -############################################################################## -# -# Copyright (C) 2013-2015 Therp BV -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## +# © 2013-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). import re -from datetime import datetime from lxml import etree -from openerp.addons.account_bank_statement_import.parserlib import ( - BankStatement) class CamtParser(object): @@ -58,7 +39,7 @@ class CamtParser(object): attr_value = found_node[0].text else: attr_value = join_str.join([x.text for x in found_node]) - setattr(obj, attr_name, attr_value) + obj[attr_name] = attr_value break def parse_transaction_details(self, ns, node, transaction): @@ -67,16 +48,21 @@ class CamtParser(object): self.add_value_from_node( ns, node, [ './ns:RmtInf/ns:Ustrd', - './ns:AddtlTxInf', './ns:AddtlNtryInf', - ], transaction, 'message', join_str='\n') + './ns:Refs/ns:InstrId', + ], transaction, 'note', join_str='\n') + # name + self.add_value_from_node( + ns, node, [ + './ns:AddtlTxInf', + ], transaction, 'name', join_str='\n') # eref self.add_value_from_node( ns, node, [ './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref', './ns:Refs/ns:EndToEndId', ], - transaction, 'eref' + transaction, 'ref' ) # remote party values party_type = 'Dbtr' @@ -88,15 +74,15 @@ class CamtParser(object): './ns:RltdPties/ns:%s' % party_type, namespaces={'ns': ns}) if party_node: self.add_value_from_node( - ns, party_node[0], './ns:Nm', transaction, 'remote_owner') + ns, party_node[0], './ns:Nm', transaction, 'partner_name') self.add_value_from_node( ns, party_node[0], './ns:PstlAdr/ns:Ctry', transaction, - 'remote_owner_country' + 'partner_country' ) address_node = party_node[0].xpath( './ns:PstlAdr/ns:AdrLine', namespaces={'ns': ns}) if address_node: - transaction.remote_owner_address = [address_node[0].text] + transaction['partner_address'] = [address_node[0].text] # Get remote_account from iban or from domestic account: account_node = node.xpath( './ns:RltdPties/ns:%sAcct/ns:Id' % party_type, @@ -106,45 +92,52 @@ class CamtParser(object): iban_node = account_node[0].xpath( './ns:IBAN', namespaces={'ns': ns}) if iban_node: - transaction.remote_account = iban_node[0].text + transaction['account_number'] = iban_node[0].text bic_node = node.xpath( './ns:RltdAgts/ns:%sAgt/ns:FinInstnId/ns:BIC' % party_type, namespaces={'ns': ns} ) if bic_node: - transaction.remote_bank_bic = bic_node[0].text + transaction['account_bic'] = bic_node[0].text else: self.add_value_from_node( ns, account_node[0], './ns:Othr/ns:Id', transaction, - 'remote_account' + 'account_number' ) - def parse_transaction(self, ns, node, transaction): + def parse_transaction(self, ns, node): """Parse transaction (entry) node.""" + transaction = {} self.add_value_from_node( ns, node, './ns:BkTxCd/ns:Prtry/ns:Cd', transaction, 'transfer_type' ) + self.add_value_from_node( + ns, node, './ns:BookgDt/ns:Dt', transaction, 'date') self.add_value_from_node( ns, node, './ns:BookgDt/ns:Dt', transaction, 'execution_date') self.add_value_from_node( ns, node, './ns:ValDt/ns:Dt', transaction, 'value_date') - transaction.transferred_amount = self.parse_amount(ns, node) + + transaction['amount'] = self.parse_amount(ns, node) + details_node = node.xpath( './ns:NtryDtls/ns:TxDtls', namespaces={'ns': ns}) if details_node: self.parse_transaction_details(ns, details_node[0], transaction) - if not transaction.message: + if not transaction.get('name'): self.add_value_from_node( - ns, node, './ns:AddtlNtryInf', transaction, 'message') - if not transaction.eref: + ns, node, './ns:AddtlNtryInf', transaction, 'name') + if not transaction.get('name'): + transaction['name'] = '/' + if not transaction.get('ref'): self.add_value_from_node( ns, node, [ './ns:NtryDtls/ns:Btch/ns:PmtInfId', ], - transaction, 'eref' + transaction, 'ref' ) - transaction.data = etree.tostring(node) + transaction['data'] = etree.tostring(node) return transaction def get_balance_amounts(self, ns, node): @@ -182,27 +175,28 @@ class CamtParser(object): def parse_statement(self, ns, node): """Parse a single Stmt node.""" - statement = BankStatement() + result = {} self.add_value_from_node( ns, node, [ './ns:Acct/ns:Id/ns:IBAN', './ns:Acct/ns:Id/ns:Othr/ns:Id', - ], statement, 'local_account' + ], result, 'account_number' ) self.add_value_from_node( - ns, node, './ns:Id', statement, 'statement_id') + ns, node, './ns:Id', result, 'name') + self.add_value_from_node( + ns, node, './ns:Dt', result, 'date') self.add_value_from_node( - ns, node, './ns:Acct/ns:Ccy', statement, 'local_currency') - (statement.start_balance, statement.end_balance) = ( + ns, node, './ns:Acct/ns:Ccy', result, 'currency') + result['balance_start'], result['balance_end_real'] = ( self.get_balance_amounts(ns, node)) transaction_nodes = node.xpath('./ns:Ntry', namespaces={'ns': ns}) + result['transactions'] = [] for entry_node in transaction_nodes: - transaction = statement.create_transaction() - self.parse_transaction(ns, entry_node, transaction) - if statement['transactions']: - statement.date = datetime.strptime( - statement['transactions'][0].execution_date, "%Y-%m-%d") - return statement + transaction = self.parse_transaction(ns, entry_node) + if transaction: + result['transactions'].append(transaction) + return result def check_version(self, ns, root): """Validate validity of camt file.""" @@ -242,8 +236,14 @@ class CamtParser(object): ns = root.tag[1:root.tag.index("}")] self.check_version(ns, root) statements = [] + currency = None + account_number = None for node in root[0][1:]: statement = self.parse_statement(ns, node) if len(statement['transactions']): + if 'currency' in statement: + currency = statement.pop('currency') + if 'account_number' in statement: + account_number = statement.pop('account_number') statements.append(statement) - return statements + return currency, account_number, statements diff --git a/account_bank_statement_import_camt/demo/demo_data.xml b/account_bank_statement_import_camt/demo/demo_data.xml index 7fc44d7..e6313a1 100644 --- a/account_bank_statement_import_camt/demo/demo_data.xml +++ b/account_bank_statement_import_camt/demo/demo_data.xml @@ -1,26 +1,15 @@ - - - - - Bank Journal - (test camt) - TBNKCAMT - bank - - - - - - - - Your Company - NL77ABNA0574908765 - - - - bank - - - - - + + + NL77ABNA0574908765 + + + + + + Bank Journal - (test camt) + TBNKCAMT + bank + + + diff --git a/account_bank_statement_import_camt/models/__init__.py b/account_bank_statement_import_camt/models/__init__.py new file mode 100644 index 0000000..44b3362 --- /dev/null +++ b/account_bank_statement_import_camt/models/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# © 2013-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from . import account_bank_statement_import diff --git a/account_bank_statement_import_camt/models/account_bank_statement_import.py b/account_bank_statement_import_camt/models/account_bank_statement_import.py new file mode 100644 index 0000000..9d711d8 --- /dev/null +++ b/account_bank_statement_import_camt/models/account_bank_statement_import.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +"""Add process_camt method to account.bank.statement.import.""" +# © 2013-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import logging +import StringIO +import zipfile +from openerp import api, models +from ..camt import CamtParser as Parser + +_logger = logging.getLogger(__name__) + + +class AccountBankStatementImport(models.TransientModel): + """Add process_camt method to account.bank.statement.import.""" + _inherit = 'account.bank.statement.import' + + @api.model + def _parse_file(self, data_file): + """Parse a CAMT053 XML file.""" + try: + parser = Parser() + _logger.debug("Try parsing with camt.") + return parser.parse(data_file) + except ValueError: + try: + with zipfile.ZipFile(StringIO.StringIO(data_file)) as data: + currency = None + account_number = None + transactions = [] + for member in data.namelist(): + currency, account_number, new = self._parse_file( + data.open(member).read() + ) + transactions.extend(new) + return currency, account_number, transactions + except (zipfile.BadZipfile, ValueError): + pass + # Not a camt file, returning super will call next candidate: + _logger.debug("Statement file was not a camt file.", + exc_info=True) + return super(AccountBankStatementImport, self)._parse_file(data_file) diff --git a/account_bank_statement_import_camt/test_files/test-camt053.xml b/account_bank_statement_import_camt/test_files/test-camt053 similarity index 100% rename from account_bank_statement_import_camt/test_files/test-camt053.xml rename to account_bank_statement_import_camt/test_files/test-camt053 diff --git a/account_bank_statement_import_camt/tests/__init__.py b/account_bank_statement_import_camt/tests/__init__.py index cd97bca..06d3af2 100644 --- a/account_bank_statement_import_camt/tests/__init__.py +++ b/account_bank_statement_import_camt/tests/__init__.py @@ -1,23 +1,5 @@ # -*- encoding: utf-8 -*- """Test import of bank statement for camt.053.""" -############################################################################## -# -# Copyright (C) 2015 Therp BV . -# -# All other contributions are (C) by their respective contributors -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## +# © 2013-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). from . import test_import_bank_statement diff --git a/account_bank_statement_import_camt/tests/test_import_bank_statement.py b/account_bank_statement_import_camt/tests/test_import_bank_statement.py index d14aca9..496ae94 100644 --- a/account_bank_statement_import_camt/tests/test_import_bank_statement.py +++ b/account_bank_statement_import_camt/tests/test_import_bank_statement.py @@ -1,52 +1,54 @@ # -*- coding: utf-8 -*- """Run test to import camt.053 import.""" -############################################################################## -# -# Copyright (C) 2015 Therp BV . -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## -from openerp.addons.account_bank_statement_import.tests import ( - TestStatementFile) +# © 2013-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import base64 +from openerp.tests.common import TransactionCase +from openerp.tools.misc import file_open -class TestImport(TestStatementFile): +class TestImport(TransactionCase): """Run test to import camt import.""" + transactions = [ + { + 'account_number': 'NL46ABNA0499998748', + 'amount': -754.25, + 'date': '2014-01-05', + 'ref': '435005714488-ABNO33052620', + }, + ] def test_statement_import(self): """Test correct creation of single statement.""" - transactions = [ - { - 'remote_account': 'NL46ABNA0499998748', - 'transferred_amount': -754.25, - 'value_date': '2014-01-05', - 'ref': '435005714488-ABNO33052620', - }, - ] - self._test_statement_import( - 'account_bank_statement_import_camt', 'test-camt053.xml', - '1234Test/1', - local_account='NL77ABNA0574908765', - start_balance=15568.27, end_balance=15121.12, - transactions=transactions - ) + action = {} + with file_open( + 'account_bank_statement_import_camt/test_files/test-camt053' + ) as testfile: + action = self.env['account.bank.statement.import'].create({ + 'data_file': base64.b64encode(testfile.read()), + }).import_file() + for statement in self.env['account.bank.statement'].browse( + action['context']['statement_ids'] + ): + self.assertTrue(any( + all( + line[key] == self.transactions[0][key] + for key in ['amount', 'date', 'ref'] + ) and + line.bank_account_id.acc_number == + self.transactions[0]['account_number'] + for line in statement.line_ids + )) def test_zip_import(self): """Test import of multiple statements from zip file.""" - self._test_statement_import( - 'account_bank_statement_import_camt', 'test-camt053.zip', - '1234Test/2', # Only name of first statement - local_account='NL77ABNA0574908765', - ) + with file_open( + 'account_bank_statement_import_camt/test_files/test-camt053.zip' + ) as testfile: + action = self.env['account.bank.statement.import'].create({ + 'data_file': base64.b64encode(testfile.read()), + }).import_file() + for statement in self.env['account.bank.statement'].browse( + action['context']['statement_ids'] + ): + self.assertTrue(statement.line_ids) diff --git a/account_bank_statement_import_camt/views/account_bank_statement_import.xml b/account_bank_statement_import_camt/views/account_bank_statement_import.xml new file mode 100644 index 0000000..1b9b265 --- /dev/null +++ b/account_bank_statement_import_camt/views/account_bank_statement_import.xml @@ -0,0 +1,15 @@ + + + + + account.bank.statement.import + + +
    +
  • CAMT
    • +
  • zipped CAMT
    • +
+ + + +