migration to 11.0

base_location_geonames_import/README.rst

@@ -1,23 +1,24 @@
 .. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
-   :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+   :target: https://www.gnu.org/licenses/agpl-3.0-standalone.html
    :alt: License: AGPL-3
 
 =============================
 Base Location Geonames Import
 =============================
 
-This module adds a wizard to import better zip entries from `Geonames <http://www.geonames.org/>`_ database.
+This module adds a wizard to import cities and/or better zip entries from
+`Geonames <http://www.geonames.org/>`_ database.
 
 Installation
 ============
 
-To install this module, you need these Python libraries: requests and
-unicodecsv.
+To install this module, you need the Python library 'requests'.
 
 Configuration
 =============
 
-To access the menu to import better zip entries from Geonames, you must add yourself to the groups *Technical features* and *Sales manager*.
+To access the menu to import better zip entries from Geonames,
+you must add yourself to the groups *Technical features* and *Sales manager*.
 
 If want want/need to modify the default URL
 (http://download.geonames.org/export/zip/), you can set the *geonames.url*
@@ -29,14 +30,19 @@ Usage
 Go to *Settings > Technical > Cities/Locations Management > Import from Geonames*,
 and click on it to open a wizard.
 
-When you start the wizard, it will ask you to select a country. Then, for the
+When you start the wizard, it will ask you to select a country. If the
+country has been set-up to require the entry of cites from a list you will be
+able to indicate if you want to import the cities or zip codes.
+
+
+Then, for the
 selected country, it will delete all the current better zip entries, download
 the latest version of the list of cities from geonames.org and create new
 better zip entries.
 
 .. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas
    :alt: Try me on Runbot
-   :target: https://runbot.odoo-community.org/runbot/134/10.0
+   :target: https://runbot.odoo-community.org/runbot/134/11.0
 
 Bug Tracker
 ===========
@@ -54,8 +60,9 @@ Contributors
 
 * Alexis de Lattre <alexis.delattre@akretion.com>
 * Lorenzo Battistini <lorenzo.battistini@agilebg.com>
-* Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
+* Pedro M. Baeza <pedro.baeza@tecnativa.com>
 * Dave Lasley <dave@laslabs.com>
+* Jordi Ballester <jordi.ballester@eficent.com>
 
 Icon
 ----

base_location_geonames_import/__manifest__.py

@@ -1,12 +1,15 @@
 # -*- coding: utf-8 -*-
-# © 2014-2016 Akretion (Alexis de Lattre <alexis.delattre@akretion.com>)
-# © 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
-# © 2016 Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
-# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+# Copyright 2014-2016 Akretion (Alexis de Lattre
+#                     <alexis.delattre@akretion.com>)
+# Copyright 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
+# Copyright 2016 Pedro M. Baeza <pedro.baeza@tecnativa.com>
+# Copyright 2017 Eficent Business and IT Consulting Services, S.L.
+#                <contact@eficent.com>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
 
 {
     'name': 'Base Location Geonames Import',
-    'version': '10.0.1.0.1',
+    'version': '11.0.1.0.1',
     'category': 'Partner Management',
     'license': 'AGPL-3',
     'summary': 'Import better zip entries from Geonames',
@@ -17,7 +20,7 @@
               'Odoo Community Association (OCA)',
     'website': 'http://www.akretion.com',
     'depends': ['base_location'],
-    'external_dependencies': {'python': ['requests', 'unicodecsv']},
+    'external_dependencies': {'python': ['requests']},
     'data': [
         'wizard/geonames_import_view.xml',
         ],

base_location_geonames_import/tests/test_base_location_geonames_import.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-# © 2016 Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
-# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+# Copyright 2016 Pedro M. Baeza <pedro.baeza@tecnativa.com>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
 
 from odoo.tests import common
 
@@ -10,6 +10,7 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
     def setUpClass(cls):
         super(TestBaseLocationGeonamesImport, cls).setUpClass()
         cls.country = cls.env.ref('base.mc')
+        cls.country.enforce_cities = True
         cls.wizard = cls.env['better.zip.geonames.import'].create({
             'country_id': cls.country.id,
         })
@@ -27,12 +28,25 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
             ('country_id', '=', self.country.id)
         ])
         self.assertEqual(zip_count, max_import)
+
+        # Look if there are imported cities
+        city_count = self.env['res.city'].search_count([
+            ('country_id', '=', self.country.id)
+        ])
+        self.assertTrue(city_count)
+
         # Reimport again to see that there's no duplicates
         self.wizard.with_context(max_import=max_import).run_import()
         state_count2 = self.env['res.country.state'].search_count([
             ('country_id', '=', self.country.id)
         ])
         self.assertEqual(state_count, state_count2)
+
+        city_count2 = self.env['res.city'].search_count([
+            ('country_id', '=', self.country.id)
+        ])
+        self.assertEqual(city_count, city_count2)
+
         zip_count = self.env['res.better.zip'].search_count([
             ('country_id', '=', self.country.id)
         ])
@@ -46,6 +60,13 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
         self.wizard.run_import()
         self.assertFalse(zip_entry.exists())
 
+        city_entry = self.env['res.city'].create({
+            'name': 'Test city',
+            'country_id': self.country.id,
+        })
+        self.wizard.run_import()
+        self.assertFalse(city_entry.exists())
+
     def test_import_title(self):
         self.wizard.letter_case = 'title'
         self.wizard.with_context(max_import=1).run_import()
@@ -54,6 +75,11 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
         )
         self.assertEqual(zip.city, zip.city.title())
 
+        city = self.env['res.city'].search(
+            [('country_id', '=', self.country.id)], limit=1
+        )
+        self.assertEqual(city.name, city.name.title())
+
     def test_import_upper(self):
         self.wizard.letter_case = 'upper'
         self.wizard.with_context(max_import=1).run_import()
@@ -61,3 +87,8 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
             [('country_id', '=', self.country.id)], limit=1
         )
         self.assertEqual(zip.city, zip.city.upper())
+
+        city = self.env['res.city'].search(
+            [('country_id', '=', self.country.id)], limit=1
+        )
+        self.assertEqual(city.name, city.name.upper())

base_location_geonames_import/wizard/geonames_import.py

@@ -1,22 +1,21 @@
 # -*- coding: utf-8 -*-
-# © 2014-2016 Akretion (Alexis de Lattre <alexis.delattre@akretion.com>)
-# © 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
-# © 2016 Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
-# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+# Copyright 2014-2016 Akretion (Alexis de Lattre
+#                     <alexis.delattre@akretion.com>)
+# Copyright 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
+# Copyright 2016 Pedro M. Baeza <pedro.baeza@tecnativa.com>
+# Copyright 2017 Eficent Business and IT Consulting Services, S.L.
+#                <contact@eficent.com>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
 
 from odoo import _, api, fields, models, tools
 from odoo.exceptions import UserError
 import requests
 import tempfile
-import StringIO
+import io
 import zipfile
 import os
 import logging
-
-try:
-    import unicodecsv
-except ImportError:
-    unicodecsv = None
+import csv
 
 logger = logging.getLogger(__name__)
 
@@ -27,6 +26,12 @@ class BetterZipGeonamesImport(models.TransientModel):
     _rec_name = 'country_id'
 
     country_id = fields.Many2one('res.country', 'Country', required=True)
+    enforce_cities = fields.Boolean(string='Enforce Cities',
+                                    help='The city will be created as a '
+                                         'separate entity.',
+                                    related='country_id.enforce_cities',
+                                    readonly=True)
+
     letter_case = fields.Selection([
         ('unchanged', 'Unchanged'),
         ('title', 'Title Case'),
@@ -46,26 +51,46 @@ class BetterZipGeonamesImport(models.TransientModel):
         return city
 
     @api.model
-    def _domain_search_better_zip(self, row, country):
-        return [('name', '=', row[1]),
-                ('city', '=', self.transform_city_name(row[2], country)),
+    def _domain_search_res_city(self, row, country):
+        return [('name', '=', self.transform_city_name(row[2], country)),
                 ('country_id', '=', country.id)]
 
     @api.model
-    def _prepare_better_zip(self, row, country):
+    def _domain_search_better_zip(self, row, country, res_city):
+        domain = [('name', '=', row[1]),
+                  ('city', '=', self.transform_city_name(row[2], country)),
+                  ('country_id', '=', country.id)]
+        if res_city:
+            domain += [('city_id', '=', res_city.id)]
+        return domain
+
+    @api.model
+    def _prepare_res_city(self, row, country):
+        state = self.select_or_create_state(row, country)
+        vals = {
+            'name': self.transform_city_name(row[2], country),
+            'state_id': state.id,
+            'country_id': country.id,
+        }
+        return vals
+
+    @api.model
+    def _prepare_better_zip(self, row, country, res_city):
         state = self.select_or_create_state(row, country)
+        city_name = self.transform_city_name(row[2], country)
         vals = {
             'name': row[1],
-            'city': self.transform_city_name(row[2], country),
+            'city_id': res_city and res_city.id or False,
+            'city': res_city and res_city.name or city_name,
             'state_id': state.id,
             'country_id': country.id,
             'latitude': row[9],
             'longitude': row[10],
-            }
+        }
         return vals
 
     @api.model
-    def create_better_zip(self, row, country):
+    def create_better_zip(self, row, country, res_city):
         if row[0] != country.code:
             raise UserError(
                 _("The country code inside the file (%s) doesn't "
@@ -81,16 +106,46 @@ class BetterZipGeonamesImport(models.TransientModel):
         if row[1] and row[2]:
             zip_model = self.env['res.better.zip']
             zips = zip_model.search(self._domain_search_better_zip(
-                row, country))
+                row, country, res_city))
             if zips:
                 return zips[0]
             else:
-                vals = self._prepare_better_zip(row, country)
+                vals = self._prepare_better_zip(row, country, res_city)
                 if vals:
+                    logger.debug('Creating res.better.zip %s' % vals['name'])
                     return zip_model.create(vals)
         else:  # pragma: no cover
             return False
 
+    @api.model
+    def create_res_city(self, row, country):
+        if row[0] != country.code:
+            raise UserError(
+                _("The country code inside the file (%s) doesn't "
+                    "correspond to the selected country (%s).")
+                % (row[0], country.code))
+        logger.debug('Processing city creation for ZIP = %s - City = %s' %
+                     (row[1], row[2]))
+        if self.letter_case == 'title':
+            row[2] = row[2].title()
+            row[3] = row[3].title()
+        elif self.letter_case == 'upper':
+            row[2] = row[2].upper()
+            row[3] = row[3].upper()
+        if row[2]:
+            res_city_model = self.env['res.city']
+            res_cities = res_city_model.search(self._domain_search_res_city(
+                row, country))
+            if res_cities:
+                return res_cities[0]
+            else:
+                vals = self._prepare_res_city(row, country)
+                if vals:
+                    logger.debug('Creating res.city %s' % vals['name'])
+                    return res_city_model.create(vals)
+        else:  # pragma: no cover
+            return False
+
     @tools.ormcache('country_id', 'code')
     def _get_state(self, country_id, code, name):
         state = self.env['res.country.state'].search(
@@ -117,6 +172,7 @@ class BetterZipGeonamesImport(models.TransientModel):
     def run_import(self):
         self.ensure_one()
         zip_model = self.env['res.better.zip']
+        res_city_model = self.env['res.city']
         country_code = self.country_id.code
         config_url = self.env['ir.config_parameter'].get_param(
             'geonames.url',
@@ -129,19 +185,32 @@ class BetterZipGeonamesImport(models.TransientModel):
                 _('Got an error %d when trying to download the file %s.')
                 % (res_request.status_code, url))
         # Store current record list
+
+        res_cities_to_delete = res_city_model
         zips_to_delete = zip_model.search(
             [('country_id', '=', self.country_id.id)])
-        f_geonames = zipfile.ZipFile(StringIO.StringIO(res_request.content))
-        tempdir = tempfile.mkdtemp(prefix='openerp')
+        if self.enforce_cities:
+            res_cities_to_delete = res_city_model.search(
+                [('country_id', '=', self.country_id.id)])
+
+        f_geonames = zipfile.ZipFile(io.BytesIO(res_request.content))
+        tempdir = tempfile.mkdtemp(prefix='odoo')
         f_geonames.extract('%s.txt' % country_code, tempdir)
         logger.info('The geonames zipfile has been decompressed')
-        data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r')
+        data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r',
+                         encoding='utf-8')
         data_file.seek(0)
-        logger.info('Starting to create the better zip entries')
+        logger.info('Starting to create the cities and/or better zip entries')
         max_import = self.env.context.get('max_import', 0)
-        reader = unicodecsv.reader(data_file, encoding='utf-8', delimiter='	')
+        reader = csv.reader(data_file, delimiter='	')
         for i, row in enumerate(reader):
-            zip_code = self.create_better_zip(row, self.country_id)
+            res_city = False
+            if self.enforce_cities:
+                res_city = self.create_res_city(row, self.country_id)
+                if res_city in res_cities_to_delete:
+                    res_cities_to_delete -= res_city
+            zip_code = self.create_better_zip(row, self.country_id,
+                                              res_city)
             if zip_code in zips_to_delete:
                 zips_to_delete -= zip_code
             if max_import and (i + 1) == max_import:
@@ -151,7 +220,11 @@ class BetterZipGeonamesImport(models.TransientModel):
             zips_to_delete.unlink()
             logger.info('%d better zip entries deleted for country %s' %
                         (len(zips_to_delete), self.country_id.name))
+        if res_cities_to_delete and not max_import:
+            res_cities_to_delete.unlink()
+            logger.info('%d res.city entries deleted for country %s' %
+                        (len(res_cities_to_delete), self.country_id.name))
         logger.info(
-            'The wizard to create better zip entries from geonames '
-            'has been successfully completed.')
+            'The wizard to create cities and/or better zip entries from '
+            'geonames has been successfully completed.')
         return True

base_location_geonames_import/wizard/geonames_import_view.xml

@@ -13,6 +13,7 @@
                 </group>
                 <group name="main">
                     <field name="country_id"/>
+                    <field name="enforce_cities"/>
                     <field name="letter_case"/>
                 </group>
                 <footer>