[11.0] [FIX] float parsing in paypal import (#211)

In English version of the files, paypal uses ',' as thousands separator and '.' as
decimal separator -> the previous code would populate the bank statement with
wrong values without complaining.

We fix this by allowing to configure the separators in the paypal map.

account_bank_statement_import_paypal/data/paypal_map_data.xml

@@ -3,6 +3,8 @@
 
     <record id="paypal_map" model="account.bank.statement.import.paypal.map">
       <field name="name">Paypal Monthly Statement</field>
+      <field name="float_thousands_sep">comma</field>
+      <field name="float_decimal_sep">dot</field>
     </record>
 
     <record id="paypal_map_line_date" model="account.bank.statement.import.paypal.map.line">

account_bank_statement_import_paypal/models/account_bank_statement_import_paypal_map.py

@@ -1,7 +1,7 @@
 # Copyright 2019 Tecnativa - Vicent Cubells
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
 
-from odoo import fields, models
+from odoo import fields, models, api
 
 
 class AccountBankStatementImportPaypalMap(models.Model):
@@ -17,6 +17,50 @@ class AccountBankStatementImportPaypalMap(models.Model):
         required=True,
         copy=True,
     )
+    float_thousands_sep = fields.Selection(
+        [('dot', 'dot (.)'),
+         ('comma', 'comma (,)'),
+         ('none', 'none'),
+         ],
+        string='Thousands separator',
+        # forward compatibility: this was the value assumed
+        # before the field was added.
+        default='dot',
+        required=True
+    )
+    float_decimal_sep = fields.Selection(
+        [('dot', 'dot (.)'),
+         ('comma', 'comma (,)'),
+         ('none', 'none'),
+         ],
+        string='Decimals separator',
+        # forward compatibility: this was the value assumed
+        # before the field was added.
+        default='comma',
+        required=True
+    )
+
+    @api.onchange('float_thousands_sep')
+    def onchange_thousands_separator(self):
+        if 'dot' == self.float_thousands_sep == self.float_decimal_sep:
+            self.float_decimal_sep = 'comma'
+        elif 'comma' == self.float_thousands_sep == self.float_decimal_sep:
+            self.float_decimal_sep = 'dot'
+
+    @api.onchange('float_decimal_sep')
+    def onchange_decimal_separator(self):
+        if 'dot' == self.float_thousands_sep == self.float_decimal_sep:
+            self.float_thousands_sep = 'comma'
+        elif 'comma' == self.float_thousands_sep == self.float_decimal_sep:
+            self.float_thousands_sep = 'dot'
+
+    def _get_separators(self):
+        separators = {'dot': '.',
+                      'comma': ',',
+                      'none': '',
+                      }
+        return (separators[self.float_thousands_sep],
+                separators[self.float_decimal_sep])
 
 
 class AccountBankStatementImportPaypalMapLIne(models.Model):

account_bank_statement_import_paypal/tests/paypal_en.csv

@@ -1,3 +1,3 @@
 "Date","Time","Time Zone","Description","Currency","Gross","Fee ","Net","Balance","Transaction ID","From Email Address","Name","Bank Name","Bank Account","Shipping and Handling Amount","Sales Tax","Invoice ID","Reference Txn ID"
-"12/15/2018","20:07:53","CET","Your best supplier","USD","-33,50","-2,3","-31,2","-31,2","53820712527632627","","John Doe","Bank of America","123456789","0","0","INV25","23"
-"12/15/2018","22:07:53","CET","Your payment","USD","525","0","525","493,80","34731322767782103","","Agrolait","","","0","0","INV/2019/0003","24"
+"12/15/2018","20:07:53","CET","Your best supplier","USD","-33.50","-2.3","-31.2","-31.2","53820712527632627","","John Doe","Bank of America","123456789","0","0","INV25","23"
+"12/15/2018","22:07:53","CET","Your payment","USD","1,525.00","0","1,525.00","1,493.80","34731322767782103","","Agrolait","","","0","0","INV/2019/0003","24"

account_bank_statement_import_paypal/tests/test_paypal_statement_import.py

@@ -42,8 +42,10 @@ class TestPaypalFile(common.SavepointCase):
         # Current statements before to run the wizard
         old_statements = self.env['account.bank.statement'].search([])
         # This journal is for Paypal statements
-        map = self.env.ref('account_bank_statement_import_paypal.paypal_map')
-        self.journal.paypal_map_id = map.id
+        paypal_map = self.env.ref(
+            'account_bank_statement_import_paypal.paypal_map'
+        )
+        self.journal.paypal_map_id = paypal_map.id
         file = self._do_import('paypal_en.csv')
         file = base64.b64encode(file.encode("utf-8"))
         wizard = self.env['account.bank.statement.import'].with_context({
@@ -55,4 +57,6 @@ class TestPaypalFile(common.SavepointCase):
         self.assertEqual(len(statement.line_ids), 3)
         self.assertEqual(len(statement.mapped('line_ids').filtered(
             lambda x: x.partner_id and x.account_id)), 1)
-        self.assertAlmostEqual(sum(statement.mapped('line_ids.amount')), 489.2)
+        self.assertAlmostEqual(
+            sum(statement.mapped('line_ids.amount')), 1489.2
+        )

account_bank_statement_import_paypal/views/paypal_map_views.xml

@@ -16,6 +16,8 @@
             <form string="Paypal Map">
                 <group>
                     <field name="name"/>
+                    <field name="float_thousands_sep"/>
+                    <field name="float_decimal_sep"/>
                 </group>
                 <separator string="Mapping Lines"/>
                 <field name="map_line_ids"/>

account_bank_statement_import_paypal/wizards/account_bank_statement_import_paypal.py

@@ -55,10 +55,13 @@ class AccountBankStatementImport(models.TransientModel):
 
     @api.model
     def _paypal_convert_amount(self, amount_str):
-        """ This method is designed to be inherited """
-        valstr = re.sub(r'[^\d,.-]', '', amount_str)
-        valstrdot = valstr.replace('.', '')
-        valstrdot = valstrdot.replace(',', '.')
+        if self.paypal_map_id:
+            thousands, decimal = self.paypal_map_id._get_separators()
+        else:
+            thousands, decimal = ',', '.'
+        valstr = re.sub(r'[^\d%s%s.-]' % (thousands, decimal), '', amount_str)
+        valstrdot = valstr.replace(thousands, '')
+        valstrdot = valstrdot.replace(decimal, '.')
         return float(valstrdot)
 
     @api.model
@@ -77,19 +80,19 @@ class AccountBankStatementImport(models.TransientModel):
     def _convert_paypal_line_to_dict(self, idx, line):
         rline = dict()
         for item in range(len(line)):
-            map = self.mapped('paypal_map_id.map_line_ids')[item]
+            paypal_map = self.mapped('paypal_map_id.map_line_ids')[item]
             value = line[item]
-            if not map.field_to_assign:
+            if not paypal_map.field_to_assign:
                 continue
-            if map.date_format:
+            if paypal_map.date_format:
                 try:
                     value = fields.Date.to_string(
-                        datetime.strptime(value, map.date_format))
+                        datetime.strptime(value, paypal_map.date_format))
                 except Exception:
                     raise UserError(
                         _("Date format of map file and Paypal date does "
                           "not match."))
-            rline[map.field_to_assign] = value
+            rline[paypal_map.field_to_assign] = value
 
         for field in ['commission', 'amount', 'balance']:
             _logger.debug('Trying to convert %s to float' % rline[field])