[RFR] Complete streamlining/refactoring of bank statement import.

bank_statement_parse module now contains only one library that can be used to convert 7.0 parsers,
or to provide a common reference for new parsers, providing some independence of the database structure for
bank statements in base Odoo.

bank_statement_parse/README.rst

@@ -4,7 +4,3 @@ modules written for earlies versions of Odoo/OpenERP.
 
 At the same time the utility classes can be used as a reference for the
 information that can be present in bank statements and/or bank transactions.
-
-RECOMMENDATION
-
-Install the web_sheet_full_width to have a good view on bank statement files.

bank_statement_parse/__init__.py

@@ -25,7 +25,6 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 ##############################################################################
-from . import model
 from . import parserlib
 
 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

bank_statement_parse/__openerp__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 ##############################################################################
 #
-#    Copyright (C) 2011-2015 Therp BV <http://therp.nl>.
+#    Copyright (C) 2015 Therp BV <http://therp.nl>.
 #
 #    All other contributions are (C) by their respective contributors
 #
@@ -28,13 +28,9 @@
     'author': 'Banking addons community',
     'website': 'https://github.com/OCA/bank-statement-import',
     'category': 'Banking addons',
-    'depends': [
-        'account_bank_statement_import',
-        ],
-    'data': [
-    ],
-    'js': [
-    ],
+    'depends': [],
+    'data': [],
+    'js': [],
     'installable': True,
     'auto_install': False,
 }

bank_statement_parse/i18n/bank_statement_parse.pot

@@ -1,91 +0,0 @@
-# Translation of Odoo Server.
-# This file contains the translation of the following modules:
-#	* bank_statement_parse
-#
-msgid ""
-msgstr ""
-"Project-Id-Version: Odoo Server 8.0\n"
-"Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2015-01-02 10:43+0000\n"
-"PO-Revision-Date: 2015-01-02 10:43+0000\n"
-"Last-Translator: <>\n"
-"Language-Team: \n"
-"MIME-Version: 1.0\n"
-"Content-Type: text/plain; charset=UTF-8\n"
-"Content-Transfer-Encoding: \n"
-"Plural-Forms: \n"
-
-#. module: bank_statement_parse
-#: model:ir.actions.act_window,name:bank_statement_parse.action_res_partner_banks
-#: model:ir.ui.menu,name:bank_statement_parse.menu_res_partner_banks
-msgid "Bank Accounts"
-msgstr ""
-
-#. module: bank_statement_parse
-#: model:ir.ui.menu,name:bank_statement_parse.menu_finance_banking_settings
-msgid "Banking"
-msgstr ""
-
-#. module: bank_statement_parse
-#: field:account.bank.statement.import,company_id:0
-msgid "Company"
-msgstr ""
-
-#. module: bank_statement_parse
-#: selection:account.bank.statement.import,state:0
-msgid "Error"
-msgstr ""
-
-#. module: bank_statement_parse
-#: selection:account.bank.statement.import,state:0
-msgid "Finished"
-msgstr ""
-
-#. module: bank_statement_parse
-#: model:ir.model,name:bank_statement_parse.model_account_bank_statement_import
-msgid "Import Bank Statement"
-msgstr ""
-
-#. module: bank_statement_parse
-#: field:account.bank.statement.import,date:0
-msgid "Import Date"
-msgstr ""
-
-#. module: bank_statement_parse
-#: field:account.bank.statement.import,log:0
-msgid "Import Log"
-msgstr ""
-
-#. module: bank_statement_parse
-#: model:ir.actions.act_window,name:bank_statement_parse.action_account_bank_statement_import_tree
-#: model:ir.ui.menu,name:bank_statement_parse.menu_account_bank_statement_import_tree
-msgid "Imported Bank Statements Files"
-msgstr ""
-
-#. module: bank_statement_parse
-#: code:addons/bank_statement_parse/parserlib/bank_transaction.py:235
-#, python-format
-msgid "Invalid value for transfer_type"
-msgstr ""
-
-#. module: bank_statement_parse
-#: selection:account.bank.statement.import,state:0
-msgid "Review"
-msgstr ""
-
-#. module: bank_statement_parse
-#: field:account.bank.statement.import,state:0
-msgid "State"
-msgstr ""
-
-#. module: bank_statement_parse
-#: code:addons/bank_statement_parse/parserlib/bank_statement_parser.py:121
-#, python-format
-msgid "This is a stub. Please implement your own."
-msgstr ""
-
-#. module: bank_statement_parse
-#: selection:account.bank.statement.import,state:0
-msgid "Unfinished"
-msgstr ""
-

bank_statement_parse/i18n/nl.po

@@ -1,92 +0,0 @@
-# Translation of Odoo Server.
-# This file contains the translation of the following modules:
-# 	* bank_statement_parse
-#
-msgid ""
-msgstr ""
-"Project-Id-Version: Odoo Server 8.0\n"
-"Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2015-01-02 10:43+0000\n"
-"PO-Revision-Date: 2015-01-02 12:09+0100\n"
-"Last-Translator: BAS Solutions <erwin@bas-solutions.nl>\n"
-"Language-Team: \n"
-"MIME-Version: 1.0\n"
-"Content-Type: text/plain; charset=UTF-8\n"
-"Content-Transfer-Encoding: 8bit\n"
-"Plural-Forms: \n"
-"Language: nl\n"
-"X-Generator: Poedit 1.7.1\n"
-
-#. module: bank_statement_parse
-#: model:ir.actions.act_window,name:bank_statement_parse.action_res_partner_banks
-#: model:ir.ui.menu,name:bank_statement_parse.menu_res_partner_banks
-msgid "Bank Accounts"
-msgstr "Bankrekeningen"
-
-#. module: bank_statement_parse
-#: model:ir.ui.menu,name:bank_statement_parse.menu_finance_banking_settings
-msgid "Banking"
-msgstr "Bankieren"
-
-#. module: bank_statement_parse
-#: field:account.bank.statement.import,company_id:0
-msgid "Company"
-msgstr "Bedrijf"
-
-#. module: bank_statement_parse
-#: selection:account.bank.statement.import,state:0
-msgid "Error"
-msgstr "Fout"
-
-#. module: bank_statement_parse
-#: selection:account.bank.statement.import,state:0
-msgid "Finished"
-msgstr "Gereed"
-
-#. module: bank_statement_parse
-#: model:ir.model,name:bank_statement_parse.model_account_bank_statement_import
-msgid "Import Bank Statement"
-msgstr "Importeer bankafschrift"
-
-#. module: bank_statement_parse
-#: field:account.bank.statement.import,date:0
-msgid "Import Date"
-msgstr "Importeerdatum"
-
-#. module: bank_statement_parse
-#: field:account.bank.statement.import,log:0
-msgid "Import Log"
-msgstr "Importeerlog"
-
-#. module: bank_statement_parse
-#: model:ir.actions.act_window,name:bank_statement_parse.action_account_bank_statement_import_tree
-#: model:ir.ui.menu,name:bank_statement_parse.menu_account_bank_statement_import_tree
-msgid "Imported Bank Statements Files"
-msgstr "Geïmporteerde bankafschriften"
-
-#. module: bank_statement_parse
-#: code:addons/bank_statement_parse/parserlib/bank_transaction.py:235
-#, python-format
-msgid "Invalid value for transfer_type"
-msgstr "Ongeldige waarde voor transfer_type"
-
-#. module: bank_statement_parse
-#: selection:account.bank.statement.import,state:0
-msgid "Review"
-msgstr "Controleren"
-
-#. module: bank_statement_parse
-#: field:account.bank.statement.import,state:0
-msgid "State"
-msgstr "Status"
-
-#. module: bank_statement_parse
-#: code:addons/bank_statement_parse/parserlib/bank_statement_parser.py:121
-#, python-format
-msgid "This is a stub. Please implement your own."
-msgstr "Dit is een stub. Importeer uw eigen."
-
-#. module: bank_statement_parse
-#: selection:account.bank.statement.import,state:0
-msgid "Unfinished"
-msgstr "Niet gereed"

bank_statement_parse/model/__init__.py

@@ -1,30 +0,0 @@
-# -*- encoding: utf-8 -*-
-"""Import python modules extending orm models."""
-##############################################################################
-#
-#    Copyright (C) 2014 Therp BV - http://therp.nl.
-#    All Rights Reserved
-#
-#    WARNING: This program as such is intended to be used by professional
-#    programmers who take the whole responsability of assessing all potential
-#    consequences resulting from its eventual inadequacies and bugs
-#    End users who are looking for a ready-to-use solution with commercial
-#    garantees and support are strongly adviced to contract EduSense BV
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as published
-#    by the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
-from . import account_bank_statement_import
-
-# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

bank_statement_parse/model/account_bank_statement_import.py

@@ -1,105 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Extend account.bank.statement.import."""
-##############################################################################
-#
-#    Copyright (C) 2015 Therp BV <http://therp.nl>.
-#
-#    All Rights Reserved
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as
-#    published by the Free Software Foundation, either version 3 of the
-#    License, or (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
-from openerp import models
-from openerp.tools.translate import _
-
-
-class AccountBankStatementImport(models.TransientModel):
-    """Import Bank Statements File."""
-    _inherit = 'account.bank.statement.import'
-    _description = __doc__
-
-    def convert_transaction(
-            self, cr, uid, transaction, context=None):
-        """Convert transaction object to values for create."""
-        partner_vals = {
-            'name': transaction.remote_owner,
-        }
-        bank_vals = {
-            'acc_number': transaction.remote_account,
-            'owner_name': transaction.remote_owner,
-            'street': transaction.remote_owner_address,
-            'city': transaction.remote_owner_city,
-            'zip': transaction.remote_owner_postalcode,
-            'country_code': transaction.remote_owner_country_code,
-            'bank_bic': transaction.remote_bank_bic,
-        }
-        bank_account_id, partner_id = self.detect_partner_and_bank(
-            cr, uid, transaction_vals=None, partner_vals=partner_vals,
-            bank_vals=bank_vals, context=context
-        )
-        vals_line = {
-            'date': transaction.value_date,
-            'name': (
-                transaction.message or transaction.eref or
-                transaction.remote_owner or ''),  # name is required
-            'ref': transaction.eref,
-            'amount': transaction.transferred_amount,
-            'partner_name': transaction.remote_owner,
-            'acc_number': transaction.remote_account,
-            'partner_id': partner_id,
-            'bank_account_id': bank_account_id,
-            'unique_import_id': transaction.transaction_id,
-        }
-        return vals_line
-
-    def convert_statements(
-            self, cr, uid, os_statements, context=None):
-        """Taking lots of code from the former import wizard, convert array
-        of BankStatement objects to values that can be used in create of
-        bank.statement model, including bank.statement.line tuple."""
-        # os_ = old style
-        # ns_ = new style
-        ns_statements = []
-        for statement in os_statements:
-            # Set statement_data
-            ns_statement = dict(
-                acc_number=statement.local_account,
-                name=statement.statement_id,
-                date=statement.date.strftime('%Y-%m-%d'),
-                balance_start=statement.start_balance,
-                balance_end_real=statement.end_balance,
-                balance_end=statement.end_balance,
-                state='draft',
-                user_id=uid,
-            )
-            ns_transactions = []
-            subno = 0
-            for transaction in statement.transactions:
-                subno += 1
-                if not transaction.transaction_id:
-                    transaction.transaction_id = (
-                        statement.statement_id + str(subno).zfill(4))
-                ns_transactions.append(
-                    self.convert_transaction(
-                        cr, uid, transaction, context=context))
-            ns_statement['transactions'] = ns_transactions
-            ns_statements.append(ns_statement)
-        return (
-            # For the moment all statements must have same currency and amount
-            ns_statements[0].local_currency,
-            ns_statements[0].local_account,
-            ns_statements,
-        )
-
-# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

bank_statement_parse/parserlib.py

@@ -21,6 +21,50 @@
 ##############################################################################
 
 
+def convert_transaction(transaction):
+    """Convert transaction object to values for create."""
+    vals_line = {
+        'date': transaction.value_date,
+        'name': (
+            transaction.message or transaction.eref or
+            transaction.remote_owner or ''),  # name is required
+        'ref': transaction.eref,
+        'amount': transaction.transferred_amount,
+        'partner_name': transaction.remote_owner,
+        'acc_number': transaction.remote_account,
+        'unique_import_id': transaction.transaction_id,
+    }
+    return vals_line
+
+
+def convert_statements(statements):
+    """Convert statement object to values for create."""
+    vals_statements = []
+    for statement in statements:
+        # Set statement_data
+        vals_statement = {
+            'currency_code': statement.local_currency,
+            'account_number': statement.local_account,
+            'name': statement.statement_id,
+            'date': statement.date.strftime('%Y-%m-%d'),
+            'balance_start': statement.start_balance,
+            'balance_end_real': statement.end_balance,
+            'balance_end': statement.end_balance,
+            'state': 'draft',
+        }
+        statement_transactions = []
+        subno = 0
+        for transaction in statement.transactions:
+            subno += 1
+            if not transaction.transaction_id:
+                transaction.transaction_id = (
+                    statement.statement_id + str(subno).zfill(4))
+            statement_transactions.append(convert_transaction(transaction))
+        vals_statement['transactions'] = statement_transactions
+        vals_statements.append(vals_statement)
+    return vals_statements
+
+
 class BankStatement(object):
     """A bank statement groups data about several bank transactions."""
 

bank_statement_parse_camt/account_bank_statement_import.py

@@ -21,6 +21,7 @@
 ##############################################################################
 import logging
 from openerp import models
+from openerp.addons.bank_statement_parse.parserlib import convert_statements
 from .camt import CamtParser as Parser
 
 
@@ -32,20 +33,15 @@ class AccountBankStatementImport(models.TransientModel):
     _inherit = 'account.bank.statement.import'
 
     def _parse_file(self, cr, uid, data_file, context=None):
-        """
-        Parse a CAMT053 XML file
-        """
+        """Parse a CAMT053 XML file."""
         parser = Parser()
         try:
             _logger.debug("Try parsing with camt.")
-            os_statements = parser.parse(data_file)
+            return convert_statements(parser.parse(data_file))
         except ValueError:
             # Not a camt file, returning super will call next candidate:
             _logger.debug("Statement file was not a camt file.")
             return super(AccountBankStatementImport, self)._parse_file(
                 cr, uid, data_file, context=context)
-        # Succesfull parse, convert to format expected
-        return self.convert_statements(
-            cr, uid, os_statements, context=context)
 
 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: