[FIX] Prevent unneeded processing and memory use in parsers.

bank_statement_parse/parserlib.py

@@ -20,74 +20,91 @@
 ##############################################################################
 
 
-def convert_transaction(transaction):
-    """Convert transaction object to values for create."""
-    vals_line = {
-        'date': transaction.value_date,
-        'name': (
-            transaction.message or transaction.eref or
-            transaction.remote_owner or ''),  # name is required
-        'ref': transaction.eref,
-        'amount': transaction.transferred_amount,
-        'partner_name': transaction.remote_owner,
-        'account_number': transaction.remote_account,
-        'unique_import_id': transaction.transaction_id,
-    }
-    return vals_line
-
-
-def convert_statements(statements):
-    """Convert statement object to values for create."""
-    vals_statements = []
-    for statement in statements:
-        # Set statement_data
-        vals_statement = {
-            'currency_code': statement.local_currency,
-            'account_number': statement.local_account,
-            'name': statement.statement_id,
-            'date': statement.date.strftime('%Y-%m-%d'),
-            'balance_start': statement.start_balance,
-            'balance_end_real': statement.end_balance,
-            'balance_end': statement.end_balance,
-            'state': 'draft',
-        }
-        statement_transactions = []
-        subno = 0
-        for transaction in statement.transactions:
-            subno += 1
-            if not transaction.transaction_id:
-                transaction.transaction_id = (
-                    statement.statement_id + str(subno).zfill(4))
-            statement_transactions.append(convert_transaction(transaction))
-        vals_statement['transactions'] = statement_transactions
-        vals_statements.append(vals_statement)
-    return vals_statements
+class BankTransaction(dict):
+    """Single transaction that is part of a bank statement."""
 
+    @property
+    def value_date(self):
+        """property getter"""
+        return self['date']
 
-class BankStatement(object):
-    """A bank statement groups data about several bank transactions."""
+    @value_date.setter
+    def value_date(self, value_date):
+        """property setter"""
+        self['date'] = value_date
 
-    def __init__(self):
-        self.statement_id = ''
-        self.local_account = ''
-        self.local_currency = ''
-        self.start_balance = 0.0
-        self.end_balance = 0.0
-        self.date = ''
-        self.transactions = []
+    @property
+    def name(self):
+        """property getter"""
+        return self['name']
 
+    @name.setter
+    def name(self, name):
+        """property setter"""
+        self['name'] = name
 
-class BankTransaction(object):
-    """Single transaction that is part of a bank statement."""
+    @property
+    def amount(self):
+        """property getter"""
+        return self['amount']
+
+    @amount.setter
+    def amount(self, amount):
+        """property setter"""
+        self['amount'] = amount
+
+    @property
+    def eref(self):
+        """property getter"""
+        return self['ref']
+
+    @eref.setter
+    def eref(self, eref):
+        """property setter"""
+        self['ref'] = eref
+        if not self.message:
+            self.name = eref
+
+    @property
+    def message(self):
+        """property getter"""
+        return self._message
+
+    @message.setter
+    def message(self, message):
+        """property setter"""
+        self._message = message
+        self.name = message
+
+    @property
+    def remote_owner(self):
+        """property getter"""
+        return self['partner_name']
+
+    @remote_owner.setter
+    def remote_owner(self, remote_owner):
+        """property setter"""
+        self['partner_name'] = remote_owner
+        if not (self.message or self.eref):
+            self.name = remote_owner
+
+    @property
+    def remote_account(self):
+        """property getter"""
+        return self['account_number']
+
+    @remote_account.setter
+    def remote_account(self, remote_account):
+        """property setter"""
+        self['account_number'] = remote_account
 
     def __init__(self):
         """Define and initialize attributes.
 
-        Does not include attributes that belong to statement.
+        Not all attributes are already used in the actual import.
         """
-        self.transaction_id = False  # Message id
+        super(BankTransaction, self).__init__()
         self.transfer_type = False  # Action type that initiated this message
-        self.eref = False  # end to end reference for transactions
         self.execution_date = False  # The posted date of the action
         self.value_date = False  # The value date of the action
         self.remote_account = False  # The account of the other party
@@ -96,7 +113,9 @@ class BankTransaction(object):
         # The exchange rate used for conversion of local_currency and
         # remote_currency
         self.transferred_amount = 0.0  # actual amount transferred
-        self.message = False  # message from the remote party
+        self.name = ''
+        self._message = False  # message from the remote party
+        self.eref = False  # end to end reference for transactions
         self.remote_owner = False  # name of the other party
         self.remote_owner_address = []  # other parties address lines
         self.remote_owner_city = False  # other parties city name
@@ -110,3 +129,100 @@ class BankTransaction(object):
         self.storno_retry = False
         # If True, make cancelled debit eligible for a next direct debit run
         self.data = ''  # Raw data from which the transaction has been parsed
+
+
+class BankStatement(dict):
+    """A bank statement groups data about several bank transactions."""
+
+    @property
+    def statement_id(self):
+        """property getter"""
+        return self['name']
+
+    def _set_transaction_ids(self):
+        """Set transaction ids to statement_id with sequence-number."""
+        subno = 0
+        for transaction in self['transactions']:
+            subno += 1
+            transaction['unique_import_id'] = (
+                self.statement_id + str(subno).zfill(4))
+
+    @statement_id.setter
+    def statement_id(self, statement_id):
+        """property setter"""
+        self['name'] = statement_id
+        self._set_transaction_ids()
+
+    @property
+    def local_account(self):
+        """property getter"""
+        return self['account_number']
+
+    @local_account.setter
+    def local_account(self, local_account):
+        """property setter"""
+        self['account_number'] = local_account
+
+    @property
+    def local_currency(self):
+        """property getter"""
+        return self['currency_code']
+
+    @local_currency.setter
+    def local_currency(self, local_currency):
+        """property setter"""
+        self['currency_code'] = local_currency
+
+    @property
+    def start_balance(self):
+        """property getter"""
+        return self['balance_start']
+
+    @start_balance.setter
+    def start_balance(self, start_balance):
+        """property setter"""
+        self['balance_start'] = start_balance
+
+    @property
+    def end_balance(self):
+        """property getter"""
+        return self['balance_end']
+
+    @end_balance.setter
+    def end_balance(self, end_balance):
+        """property setter"""
+        self['balance_end'] = end_balance
+        self['balance_end_real'] = end_balance
+
+    @property
+    def date(self):
+        """property getter"""
+        return self['date']
+
+    @date.setter
+    def date(self, date):
+        """property setter"""
+        self['date'] = date
+
+    def create_transaction(self):
+        """Create and append transaction.
+
+        This should only be called after statement_id has been set, because
+        statement_id will become part of the unique transaction_id.
+        """
+        transaction = BankTransaction()
+        self['transactions'].append(transaction)
+        # Fill default id, but might be overruled
+        transaction['unique_import_id'] = (
+            self.statement_id + str(len(self['transactions'])).zfill(4))
+        return transaction
+
+    def __init__(self):
+        super(BankStatement, self).__init__()
+        self['transactions'] = []
+        self.statement_id = ''
+        self.local_account = ''
+        self.local_currency = ''
+        self.date = ''
+        self.start_balance = 0.0
+        self.end_balance = 0.0

bank_statement_parse_camt/account_bank_statement_import.py

@@ -20,11 +20,10 @@
 ##############################################################################
 import logging
 from openerp import models
-from openerp.addons.bank_statement_parse.parserlib import convert_statements
 from .camt import CamtParser as Parser
 
 
-_logger = logging.getLogger(__name__)
+_LOGGER = logging.getLogger(__name__)
 
 
 class AccountBankStatementImport(models.TransientModel):
@@ -35,10 +34,10 @@ class AccountBankStatementImport(models.TransientModel):
         """Parse a CAMT053 XML file."""
         parser = Parser()
         try:
-            _logger.debug("Try parsing with camt.")
-            return convert_statements(parser.parse(data_file))
+            _LOGGER.debug("Try parsing with camt.")
+            return parser.parse(data_file)
         except ValueError:
             # Not a camt file, returning super will call next candidate:
-            _logger.debug("Statement file was not a camt file.")
+            _LOGGER.debug("Statement file was not a camt file.")
             return super(AccountBankStatementImport, self)._parse_file(
                 cr, uid, data_file, context=context)

bank_statement_parse_camt/camt.py

@@ -21,10 +21,7 @@
 import re
 from datetime import datetime
 from lxml import etree
-from openerp.addons.bank_statement_parse.parserlib import (
-    BankStatement,
-    BankTransaction
-)
+from openerp.addons.bank_statement_parse.parserlib import BankStatement
 
 
 class CamtParser(object):
@@ -121,9 +118,8 @@ class CamtParser(object):
                     'remote_account'
                 )
 
-    def parse_transaction(self, ns, node):
+    def parse_transaction(self, ns, node, transaction):
         """Parse transaction (entry) node."""
-        transaction = BankTransaction()
         self.add_value_from_node(
             ns, node, './ns:BkTxCd/ns:Prtry/ns:Cd', transaction,
             'transfer_type'
@@ -190,11 +186,11 @@ class CamtParser(object):
             self.get_balance_amounts(ns, node))
         transaction_nodes = node.xpath('./ns:Ntry', namespaces={'ns': ns})
         for entry_node in transaction_nodes:
-            transaction = self.parse_transaction(ns, entry_node)
-            statement.transactions.append(transaction)
-        if statement.transactions:
+            transaction = statement.create_transaction()
+            self.parse_transaction(ns, entry_node, transaction)
+        if statement['transactions']:
             statement.date = datetime.strptime(
-                statement.transactions[0].execution_date, "%Y-%m-%d")
+                statement['transactions'][0].execution_date, "%Y-%m-%d")
         return statement
 
     def check_version(self, ns, root):
@@ -237,6 +233,6 @@ class CamtParser(object):
         statements = []
         for node in root[0][1:]:
             statement = self.parse_statement(ns, node)
-            if len(statement.transactions):
+            if len(statement['transactions']):
                 statements.append(statement)
         return statements

bank_statement_parse_mt940/mt940.py

@@ -22,7 +22,7 @@ import re
 import logging
 from datetime import datetime
 
-from openerp.addons.bank_statement_parse import parserlib
+from openerp.addons.bank_statement_parse.parserlib import BankStatement
 
 
 def str2amount(sign, amount_str):
@@ -138,8 +138,7 @@ class MT940(object):
                     self.handle_header(line, iterator)
                 line = iterator.next()
                 if not self.is_tag(line) and not self.is_footer(line):
-                    record_line = self.append_continuation_line(
-                        record_line, line)
+                    record_line += line
                     continue
                 if record_line:
                     self.handle_record(record_line)
@@ -158,19 +157,6 @@ class MT940(object):
             self.current_statement = None
         return self.statements
 
-    def append_continuation_line(self, line, continuation_line):
-        """append a continuation line for a multiline record.
-        Override and do data cleanups as necessary."""
-        return line + continuation_line
-
-    def create_statement(self):
-        """create a BankStatement."""
-        return parserlib.BankStatement()
-
-    def create_transaction(self):
-        """Create and return BankTransaction object."""
-        return parserlib.BankTransaction()
-
     def is_footer(self, line):
         """determine if a line is the footer of a statement"""
         return line and bool(re.match(self.footer_regex, line))
@@ -183,7 +169,7 @@ class MT940(object):
         """skip header lines, create current statement"""
         for dummy_i in range(self.header_lines):
             iterator.next()
-        self.current_statement = self.create_statement()
+        self.current_statement = BankStatement()
 
     def handle_footer(self, line, iterator):
         """add current statement to list, reset state"""
@@ -226,8 +212,7 @@ class MT940(object):
 
     def handle_tag_61(self, data):
         """get transaction values"""
-        transaction = self.create_transaction()
-        self.current_statement.transactions.append(transaction)
+        transaction = self.current_statement.create_transaction()
         self.current_transaction = transaction
         transaction.execution_date = datetime.strptime(data[:6], '%y%m%d')
         transaction.value_date = datetime.strptime(data[:6], '%y%m%d')

bank_statement_parse_nl_ing_mt940/account_bank_statement_import.py

@@ -20,11 +20,10 @@
 ##############################################################################
 import logging
 from openerp import models
-from openerp.addons.bank_statement_parse.parserlib import convert_statements
 from .mt940 import MT940Parser as Parser
 
 
-_logger = logging.getLogger(__name__)
+_LOGGER = logging.getLogger(__name__)
 
 
 class AccountBankStatementImport(models.TransientModel):
@@ -35,10 +34,10 @@ class AccountBankStatementImport(models.TransientModel):
         """Parse a MT940 IBAN ING file."""
         parser = Parser()
         try:
-            _logger.debug("Try parsing with MT940 IBAN ING.")
-            return convert_statements(parser.parse(data_file))
+            _LOGGER.debug("Try parsing with MT940 IBAN ING.")
+            return parser.parse(data_file)
         except ValueError:
             # Returning super will call next candidate:
-            _logger.debug("Statement file was not a MT940 IBAN ING file.")
+            _LOGGER.debug("Statement file was not a MT940 IBAN ING file.")
             return super(AccountBankStatementImport, self)._parse_file(
                 cr, uid, data_file, context=context)

bank_statement_parse_nl_ing_mt940/tests/test_import_bank_statement.py

@@ -63,7 +63,7 @@ class TestStatementFile(TransactionCase):
             (statement_obj.balance_start, start_balance)
         )
         self.assertTrue(
-            abs(statement_obj.balance_end - end_balance) < 0.00001,
+            abs(statement_obj.balance_end_real - end_balance) < 0.00001,
             'Start balance %f not equal to expected %f' %
             (statement_obj.balance_end_real, end_balance)
         )

bank_statement_parse_nl_rabo_mt940/account_bank_statement_import.py

@@ -19,12 +19,12 @@
 #
 ##############################################################################
 import logging
+
 from openerp import models
-from openerp.addons.bank_statement_parse.parserlib import convert_statements
 from .mt940 import MT940Parser as Parser
 
 
-_logger = logging.getLogger(__name__)
+_LOGGER = logging.getLogger(__name__)
 
 
 class AccountBankStatementImport(models.TransientModel):
@@ -35,10 +35,10 @@ class AccountBankStatementImport(models.TransientModel):
         """Parse a MT940 RABO file."""
         parser = Parser()
         try:
-            _logger.debug("Try parsing with MT940 RABO.")
-            return convert_statements(parser.parse(data_file))
+            _LOGGER.debug("Try parsing with MT940 RABO.")
+            return parser.parse(data_file)
         except ValueError:
             # Returning super will call next candidate:
-            _logger.debug("Statement file was not a MT940 RABO file.")
+            _LOGGER.debug("Statement file was not a MT940 RABO file.")
             return super(AccountBankStatementImport, self)._parse_file(
                 cr, uid, data_file, context=context)