refactor paypal import in order to use new bank statement from Paypal

Remove deprecated information in readme
solve issue on currency the main currency is not the first line (line are ordered by currency name). So we should take the journal currency or company currency to know the right currency to use. Also add full header to avoid issue if the paypal header change

account_bank_statement_import_paypal/README.rst

@@ -3,14 +3,6 @@ Import Paypal Bank Statements
 
 This module allows you to import the Paypal CSV files in Odoo as bank statements.
 
-Installation
-============
-
-This module depends on the module *account_bank_statement_import* which
-is available:
-* for Odoo version 8: in the OCA project `bank-statement-import <https://github.com/OCA/bank-statement-import>`
-* for Odoo master (future version 9): it is an official module.
-
 Configuration
 =============
 
@@ -19,6 +11,15 @@ In the menu Accounting > Configuration > Accounts > Setup your Bank Accounts, ma
 * Account Number: the email address associated with your Paypal account
 * Account Journal: the journal associated to your Paypal account
 
+============
+
+Go to Paypal and download your Bank Statement
+
+.. image:: account_bank_statement_import_paypal/static/description/paypal_backoffice.png
+    :alt: .
+.. image:: static/description/paypal_backoffice.png
+    :alt: .
+
 Credits
 =======
 
@@ -26,6 +27,13 @@ Contributors
 ------------
 
 * Alexis de Lattre <alexis.delattre@akretion.com>
+* Sebastien BEAU <sebastien.beau@akretion.com>
+
+TIPS
+--------
+For now only French and English report are supported
+For adding new support you just need to add your header in model/account_bank_statement_import_paypal.py in the variables HEADERS.
+Please help us and do a PR for adding new header ! Thanks
 
 Maintainer
 ----------

account_bank_statement_import_paypal/__init__.py

@@ -1,23 +1,3 @@
-# -*- encoding: utf-8 -*-
-##############################################################################
-#
-#    account_bank_statement_import_paypal module for Odoo
-#    Copyright (C) 2014-2015 Akretion (http://www.akretion.com)
-#    @author Alexis de Lattre <alexis.delattre@akretion.com>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as
-#    published by the Free Software Foundation, either version 3 of the
-#    License, or (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
+# -*- coding: utf-8 -*-
 
-from . import account_bank_statement_import_paypal
+from . import models

account_bank_statement_import_paypal/__manifest__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014-2017 Akretion (http://www.akretion.com).
+# @author Alexis de Lattre <alexis.delattre@akretion.com>
+# @author Sébastien BEAU <sebastien.beau@akretion.com>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+{
+    "name": "Import Paypal Bank Statements",
+    'summary': 'Import Paypal CSV files as Bank Statements in Odoo',
+    "version": "10.0.1.0.0",
+    "category": "Accounting",
+    "website": "https://github.com/OCA/bank-statement-import",
+    "author": " Akretion, Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "application": False,
+    "installable": True,
+    "external_dependencies": {
+        'python': ['unicodecsv'],
+        "bin": [],
+    },
+    "depends": [
+        "account_bank_statement_import",
+    ],
+    "data": [
+    ],
+    "demo": [
+    ],
+    "qweb": [
+    ]
+}

account_bank_statement_import_paypal/__openerp__.py

@@ -1,34 +0,0 @@
-# -*- encoding: utf-8 -*-
-##############################################################################
-#
-#    account_bank_statement_import_paypal module for Odoo
-#    Copyright (C) 2014-2015 Akretion (http://www.akretion.com)
-#    @author Alexis de Lattre <alexis.delattre@akretion.com>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as
-#    published by the Free Software Foundation, either version 3 of the
-#    License, or (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
-
-{
-    'name': 'Import Paypal Bank Statements',
-    'version': '0.1',
-    'license': 'AGPL-3',
-    'author': 'Akretion',
-    'website': 'http://www.akretion.com',
-    'summary': 'Import Paypal CSV files as Bank Statements in Odoo',
-    'depends': ['account_bank_statement_import'],
-    'external_dependencies': {'python': ['unicodecsv']},
-    'data': [],
-    'installable': True,
-}

account_bank_statement_import_paypal/account_bank_statement_import_paypal.py

@@ -1,236 +0,0 @@
-# -*- encoding: utf-8 -*-
-##############################################################################
-#
-#    account_bank_statement_import_paypal module for Odoo
-#    Copyright (C) 2014-2015 Akretion (http://www.akretion.com)
-#    @author Alexis de Lattre <alexis.delattre@akretion.com>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as
-#    published by the Free Software Foundation, either version 3 of the
-#    License, or (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
-
-import logging
-from datetime import datetime
-from openerp import models, fields, api, _
-from openerp.exceptions import Warning
-import unicodecsv
-import re
-from cStringIO import StringIO
-
-_logger = logging.getLogger(__name__)
-
-
-class AccountBankStatementImport(models.TransientModel):
-    _inherit = 'account.bank.statement.import'
-
-    @api.model
-    def _prepare_paypal_encoding(self):
-        '''This method is designed to be inherited'''
-        return 'utf-8'
-
-    @api.model
-    def _prepare_paypal_date_format(self):
-        '''This method is designed to be inherited'''
-        return '%d/%m/%Y'
-
-    @api.model
-    def _valid_paypal_line(self, line):
-        '''This method is designed to be inherited'''
-        col_name = line[5].replace('"','')
-        if col_name.startswith('Termin') or col_name.startswith('Rembours'):
-            return True
-        else:
-            return False
-
-    @api.model
-    def _paypal_convert_amount(self, amount_str):
-        '''This method is designed to be inherited'''
-        valstr = re.sub(r'[^\d,.-]', '', amount_str)
-        valstrdot = valstr.replace('.', '')
-        valstrdot = valstrdot.replace(',', '.')
-        return float(valstrdot)
-
-    @api.model
-    def _check_paypal(self, data_file):
-        '''This method is designed to be inherited'''
-        paypal = data_file.strip().startswith('Date,')
-        if not paypal:
-            paypal = data_file.strip().startswith('"Date",')
-
-        return paypal
-
-    @api.model
-    def _parse_file(self, data_file):
-        """ Import a file in Paypal CSV format"""
-        data_file = data_file.replace("\xef\xbb\xbf", "")
-        paypal = self._check_paypal(data_file)
-        if not paypal:
-            return super(AccountBankStatementImport, self)._parse_file(
-                data_file)
-        f = StringIO()
-        f.write(data_file)
-        f.seek(0)
-        transactions = []
-        i = 0
-        start_balance = end_balance = start_date_str = end_date_str = False
-        vals_line = False
-        company_currency_name = self.env.user.company_id.currency_id.name
-        commission_total = 0.0
-        raw_lines = []
-        paypal_email_account = False
-        # To confirm : is the encoding always latin1 ?
-        for line in unicodecsv.reader(
-                f, encoding=self._prepare_paypal_encoding()):
-            i += 1
-            _logger.debug("Line %d: %s" % (i, line))
-            if i == 1:
-                _logger.debug('Skip header line')
-                continue
-            if not line:
-                continue
-            if not self._valid_paypal_line(line):
-                _logger.info(
-                    'Skipping line %d because it is not in Done state' % i)
-                continue
-            date_dt = datetime.strptime(
-                line[0], self._prepare_paypal_date_format())
-            rline = {
-                'date': fields.Date.to_string(date_dt),
-                'currency': line[6],
-                'owner_name': line[3],
-                'amount': line[7],
-                'commission': line[8],
-                'balance': line[27],
-                'transac_ref': line[23],
-                'ref': line[12],
-                'line_nr': i,
-            }
-            for field in ['commission', 'amount', 'balance']:
-                _logger.debug('Trying to convert %s to float' % rline[field])
-                try:
-                    rline[field] = self._paypal_convert_amount(rline[field])
-                except:
-                    raise Warning(
-                        _("Value '%s' for the field '%s' on line %d, "
-                            "cannot be converted to float")
-                        % (rline[field], field, i))
-            if rline['amount'] > 0:
-                rline['name'] = line[3] + ' ' + line[10]
-                rline['partner_email'] = line[10]
-                if not paypal_email_account:
-                    paypal_email_account = line[11]
-            else:
-                rline['name'] = line[3] + ' ' + line[11]
-                rline['partner_email'] = line[11]
-                if not paypal_email_account:
-                    paypal_email_account = line[10]
-            raw_lines.append(rline)
-
-        # Second pass to sort out the lines in other currencies
-        final_lines = []
-        other_currency_line = {}
-        for wline in raw_lines:
-            if company_currency_name != wline['currency']:
-                if wline['transac_ref'] and not other_currency_line:
-                    currencies = self.env['res.currency'].search(
-                        [('name', '=', wline['currency'])])
-                    if not currencies:
-                        raise Warning(
-                            _('Currency %s on line %d cannot be found in Odoo')
-                            % (wline['currency'], wline['line_nr']))
-                    other_currency_line = {
-                        'amount_currency': wline['amount'],
-                        'currency_id': currencies[0].id,
-                        'currency': wline['currency'],
-                        'name': wline['name'],
-                        'owner_name': wline['owner_name'],
-                        'transac_ref': wline['transac_ref'],
-                    }
-
-                if other_currency_line and not wline['transac_ref']:
-                    assert (
-                        wline['currency'] == other_currency_line['currency']),\
-                        'WRONG currency'
-                    assert (
-                        wline['amount'] ==
-                        other_currency_line['amount_currency'] * -1),\
-                        'WRONG amount'
-                if (
-                        other_currency_line and
-                        wline['ref'] ==
-                        other_currency_line['transac_ref']):
-                    # reset other_currency_line
-                    other_currency_line = {}
-            else:
-                if (
-                        other_currency_line and
-                        wline['transac_ref'] ==
-                        other_currency_line['transac_ref']):
-                    wline.update(other_currency_line)
-                    # reset other_currency_line
-                    other_currency_line = {}
-                final_lines.append(wline)
-
-        # PayPal statements start with the end !
-        final_lines.reverse()
-        j = 0
-        for fline in final_lines:
-            j += 1
-            commission_total += fline['commission']
-
-            if j == 1:
-                start_date_str = fline['date']
-                start_balance = fline['balance'] - fline['amount']
-            end_date_str = fline['date']
-            end_balance = fline['balance']
-            partners = False
-            if fline['partner_email']:
-                partners = self.env['res.partner'].search(
-                    [('email', '=', fline['partner_email'])])
-            if partners:
-                partner_id = partners[0].commercial_partner_id.id
-            else:
-                partner_id = False
-            vals_line = {
-                'date': fline['date'],
-                'name': fline['ref'],
-                'ref': fline['name'],
-                'unique_import_id': fline['ref'],
-                'amount': fline['amount'],
-                'partner_id': partner_id,
-                'bank_account_id': False,
-                'currency_id': fline.get('currency_id'),
-                'amount_currency': fline.get('amount_currency'),
-                }
-            _logger.debug("vals_line = %s" % vals_line)
-            transactions.append(vals_line)
-
-        if commission_total:
-            commission_line = {
-                'date': end_date_str,
-                'name': _('Paypal commissions'),
-                'ref': _('PAYPAL-COSTS'),
-                'amount': commission_total,
-                'unique_import_id': False,
-                }
-            transactions.append(commission_line)
-
-        vals_bank_statement = {
-            'name': _('PayPal Import %s > %s')
-            % (start_date_str, end_date_str),
-            'balance_start': start_balance,
-            'balance_end_real': end_balance,
-            'transactions': transactions,
-        }
-        return None, None, [vals_bank_statement]

account_bank_statement_import_paypal/models/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import account_bank_statement_import_paypal

account_bank_statement_import_paypal/models/account_bank_statement_import_paypal.py

@@ -0,0 +1,275 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014-2017 Akretion (http://www.akretion.com).
+# @author Alexis de Lattre <alexis.delattre@akretion.com>
+# @author Sébastien BEAU <sebastien.beau@akretion.com>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+import logging
+from datetime import datetime
+from openerp import models, fields, api, _
+from openerp.exceptions import UserError
+import re
+from cStringIO import StringIO
+_logger = logging.getLogger(__name__)
+
+try:
+    import unicodecsv
+except (ImportError, IOError) as err:
+    _logger.debug(err)
+
+# Paypal header depend of the country the order are the same but the
+# value are translated. You can add you header here
+
+HEADERS = [
+    # French
+    '"Date","Heure","Fuseau horaire","Description","Devise","Avant commission"'
+    ',"Commission","Net","Solde","Numéro de transaction","Adresse email de '
+    'l\'expéditeur","Nom","Nom de la banque","Compte bancaire","Montant des '
+    'frais de livraison et de traitement","TVA","Identifiant de facture",'
+    '"Numéro de la transaction de référence"',
+    # English
+    '"Date","Time","Time Zone","Description","Currency","Gross ","Fee ","Net",'
+    '"Balance","Transaction ID","From Email Address","Name","Bank Name",'
+    '"Bank Account","Shipping and Handling Amount","Sales Tax","Invoice ID",'
+    '"Reference Txn ID"'
+    ]
+
+
+class AccountBankStatementImport(models.TransientModel):
+    _inherit = 'account.bank.statement.import'
+
+    @api.model
+    def _get_paypal_encoding(self):
+        return 'utf-8'
+
+    @api.model
+    def _get_paypal_date_format(self):
+        '''This method is designed to be inherited'''
+        return '%d/%m/%Y'
+
+    @api.model
+    def _paypal_convert_amount(self, amount_str):
+        '''This method is designed to be inherited'''
+        valstr = re.sub(r'[^\d,.-]', '', amount_str)
+        valstrdot = valstr.replace('.', '')
+        valstrdot = valstrdot.replace(',', '.')
+        return float(valstrdot)
+
+    @api.model
+    def _check_paypal(self, data_file):
+        for header in HEADERS:
+            if data_file.strip().startswith(header):
+                return True
+        return False
+
+    def _convert_paypal_line_to_dict(self, idx, line):
+        date_dt = datetime.strptime(line[0], self._get_paypal_date_format())
+        rline = {
+            'date': fields.Date.to_string(date_dt),
+            'time': line[1],
+            'description': line[3],
+            'currency': line[4],
+            'amount': line[5],
+            'commission': line[6],
+            'balance': line[8],
+            'transaction_id': line[9],
+            'email': line[10],
+            'partner_name': line[11],
+            # This two field are usefull for bank transfert
+            'bank_name': line[12],
+            'bank_account': line[13],
+            'invoice_number': line[16],
+            'origin_transaction_id': line[17],
+            'idx': idx,
+        }
+        for field in ['commission', 'amount', 'balance']:
+            _logger.debug('Trying to convert %s to float' % rline[field])
+            try:
+                rline[field] = self._paypal_convert_amount(rline[field])
+            except:
+                raise UserError(
+                    _("Value '%s' for the field '%s' on line %d, "
+                        "cannot be converted to float")
+                    % (rline[field], field, idx))
+        return rline
+
+    def _parse_paypal_file(self, data_file):
+        f = StringIO()
+        f.write(data_file)
+        f.seek(0)
+        raw_lines = []
+        reader = unicodecsv.reader(f, encoding=self._get_paypal_encoding())
+        reader.next()  # Drop header
+        for idx, line in enumerate(reader):
+            _logger.debug("Line %d: %s" % (idx, line))
+            raw_lines.append(self._convert_paypal_line_to_dict(idx, line))
+        return raw_lines
+
+    def _prepare_paypal_currency_vals(self, cline):
+        currencies = self.env['res.currency'].search(
+            [('name', '=', cline['currency'])])
+        if not currencies:
+            raise UserError(
+                _('currency %s on line %d cannot be found in odoo')
+                % (cline['currency'], cline['idx']))
+        return {
+            'amount_currency': cline['amount'],
+            'currency_id': currencies.id,
+            'currency': cline['currency'],
+            'partner_name': cline['partner_name'],
+            'description': cline['description'],
+            'email': cline['email'],
+            'transaction_id': cline['transaction_id'],
+            }
+
+    def _post_process_statement_line(self, raw_lines):
+        journal_id = self.env.context.get('journal_id')
+        if not journal_id:
+            raise UserError(_('You must run this wizard from the journal'))
+        journal = self.env['account.journal'].browse(journal_id)
+        currency = journal.currency_id or journal.company_id.currency_id
+        currency_change_lines = {}
+        real_transactions = []
+        for line in raw_lines:
+            if line['currency'] != currency.name:
+                currency_change_lines[line['transaction_id']] = line
+            else:
+                real_transactions.append(line)
+
+        for line in real_transactions:
+            # Check if the current transaction is linked with a
+            # transaction of currency change if yes merge the transaction
+            # as for odoo it's only one line
+            cline = currency_change_lines.get(line['origin_transaction_id'])
+            if cline:
+                # we update the current line with currency information
+                vals = self._prepare_paypal_currency_vals(cline)
+                line.update(vals)
+        return real_transactions
+
+    def _prepare_paypal_statement_line(self, fline):
+        if fline['bank_name']:
+            name = '|'.join([
+                fline['description'],
+                fline['bank_name'],
+                fline['bank_account']
+                ])
+        else:
+            name = '|'.join([
+                fline['description'],
+                fline['partner_name'],
+                fline['email'],
+                fline['invoice_number'],
+                ])
+        return {
+            'date': fline['date'],
+            'name': name,
+            'ref': fline['transaction_id'],
+            'unique_import_id':
+                fline['transaction_id'] + fline['date'] + fline['time'],
+            'amount': fline['amount'],
+            'bank_account_id': False,
+            'currency_id': fline.get('currency_id'),
+            'amount_currency': fline.get('amount_currency'),
+            }
+
+    def _prepare_paypal_statement(self, lines):
+        return {
+            'name':
+                _('PayPal Import %s > %s')
+                % (lines[0]['date'], lines[-1]['date']),
+            'date': lines[-1]['date'],
+            'balance_start':
+                lines[0]['balance'] -
+                lines[0]['amount'] -
+                lines[0]['commission'],
+            'balance_end_real': lines[-1]['balance'],
+            }
+
+    @api.model
+    def _parse_file(self, data_file):
+        """ Import a file in Paypal CSV format"""
+        paypal = self._check_paypal(data_file)
+        if not paypal:
+            return super(AccountBankStatementImport, self)._parse_file(
+                data_file)
+
+        raw_lines = self._parse_paypal_file(data_file)
+        final_lines = self._post_process_statement_line(raw_lines)
+
+        vals_bank_statement = self._prepare_paypal_statement(final_lines)
+
+        transactions = []
+        commission_total = 0
+        for fline in final_lines:
+            commission_total += fline['commission']
+            vals_line = self._prepare_paypal_statement_line(fline)
+            _logger.debug("vals_line = %s" % vals_line)
+            transactions.append(vals_line)
+
+        if commission_total:
+            commission_line = {
+                'date': vals_bank_statement['date'],
+                'name': _('Paypal commissions'),
+                'ref': _('PAYPAL-COSTS'),
+                'amount': commission_total,
+                'unique_import_id': False,
+                }
+            transactions.append(commission_line)
+
+        vals_bank_statement['transactions'] = transactions
+        return None, None, [vals_bank_statement]
+
+    @api.model
+    def _get_paypal_partner(self, description, partner_name,
+                            partner_email, invoice_number):
+        if invoice_number:
+            # In most case e-commerce case invoice_number
+            # will contain the sale order number
+            sale = self.env['sale.order'].search([
+                ('name', '=', invoice_number)])
+            if sale and len(sale) == 1:
+                return sale.partner_id.commercial_partner_id
+
+            invoice = self.env['account.invoice'].search([
+                ('number', '=', invoice_number)])
+            if invoice and len(invoice) == 1:
+                return invoice.partner_id.commercial_partner_id
+
+        if partner_email:
+            partner = self.env['res.partner'].search([
+                ('email', '=', partner_email),
+                ('parent_id', '=', False)])
+            if partner and len(partner) == 1:
+                return partner.commercial_partner_id
+
+        if partner_name:
+            partner = self.env['res.partner'].search([
+                ('name', '=ilike', partner_name)])
+            if partner and len(partner) == 1:
+                return partner.commercial_partner_id
+        return None
+
+    @api.model
+    def _complete_paypal_statement_line(self, line):
+        _logger.debug('Process line %s', line['name'])
+        info = line['name'].split('|')
+        if len(info) == 4:
+            partner = self._get_paypal_partner(*info)
+            if partner:
+                return {
+                    'partner_id': partner.id,
+                    'account_id': partner.property_account_receivable.id,
+                    }
+        return None
+
+    @api.model
+    def _complete_statement(self, stmts_vals, journal_id, account_number):
+        '''Match the partner from paypal information'''
+        stmts_vals = super(AccountBankStatementImport, self).\
+            _complete_statement(stmts_vals, journal_id, account_number)
+        for line in stmts_vals['transactions']:
+            vals = self._complete_paypal_statement_line(line)
+            if vals:
+                line.update(vals)
+        return stmts_vals