From 66b9d77a4f6f4a1ee385b35b7bb6a31aa0d215e2 Mon Sep 17 00:00:00 2001 From: Jordi Ballester Date: Fri, 13 Oct 2017 14:08:15 +0200 Subject: [PATCH] migration to 11.0 --- base_location_geonames_import/README.rst | 23 ++-- base_location_geonames_import/__manifest__.py | 15 ++- .../test_base_location_geonames_import.py | 35 ++++- .../wizard/geonames_import.py | 127 ++++++++++++++---- .../wizard/geonames_import_view.xml | 1 + 5 files changed, 158 insertions(+), 43 deletions(-) diff --git a/base_location_geonames_import/README.rst b/base_location_geonames_import/README.rst index bab8b798c..c37fdadeb 100644 --- a/base_location_geonames_import/README.rst +++ b/base_location_geonames_import/README.rst @@ -1,23 +1,24 @@ .. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg - :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :target: https://www.gnu.org/licenses/agpl-3.0-standalone.html :alt: License: AGPL-3 ============================= Base Location Geonames Import ============================= -This module adds a wizard to import better zip entries from `Geonames `_ database. +This module adds a wizard to import cities and/or better zip entries from +`Geonames `_ database. Installation ============ -To install this module, you need these Python libraries: requests and -unicodecsv. +To install this module, you need the Python library 'requests'. Configuration ============= -To access the menu to import better zip entries from Geonames, you must add yourself to the groups *Technical features* and *Sales manager*. +To access the menu to import better zip entries from Geonames, +you must add yourself to the groups *Technical features* and *Sales manager*. If want want/need to modify the default URL (http://download.geonames.org/export/zip/), you can set the *geonames.url* @@ -29,14 +30,19 @@ Usage Go to *Settings > Technical > Cities/Locations Management > Import from Geonames*, and click on it to open a wizard. -When you start the wizard, it will ask you to select a country. Then, for the +When you start the wizard, it will ask you to select a country. If the +country has been set-up to require the entry of cites from a list you will be +able to indicate if you want to import the cities or zip codes. + + +Then, for the selected country, it will delete all the current better zip entries, download the latest version of the list of cities from geonames.org and create new better zip entries. .. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas :alt: Try me on Runbot - :target: https://runbot.odoo-community.org/runbot/134/10.0 + :target: https://runbot.odoo-community.org/runbot/134/11.0 Bug Tracker =========== @@ -54,8 +60,9 @@ Contributors * Alexis de Lattre * Lorenzo Battistini -* Pedro M. Baeza +* Pedro M. Baeza * Dave Lasley +* Jordi Ballester Icon ---- diff --git a/base_location_geonames_import/__manifest__.py b/base_location_geonames_import/__manifest__.py index d8f4d4d4e..4462f7a99 100644 --- a/base_location_geonames_import/__manifest__.py +++ b/base_location_geonames_import/__manifest__.py @@ -1,12 +1,15 @@ # -*- coding: utf-8 -*- -# © 2014-2016 Akretion (Alexis de Lattre ) -# © 2014 Lorenzo Battistini -# © 2016 Pedro M. Baeza -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +# Copyright 2014-2016 Akretion (Alexis de Lattre +# ) +# Copyright 2014 Lorenzo Battistini +# Copyright 2016 Pedro M. Baeza +# Copyright 2017 Eficent Business and IT Consulting Services, S.L. +# +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). { 'name': 'Base Location Geonames Import', - 'version': '10.0.1.0.1', + 'version': '11.0.1.0.1', 'category': 'Partner Management', 'license': 'AGPL-3', 'summary': 'Import better zip entries from Geonames', @@ -17,7 +20,7 @@ 'Odoo Community Association (OCA)', 'website': 'http://www.akretion.com', 'depends': ['base_location'], - 'external_dependencies': {'python': ['requests', 'unicodecsv']}, + 'external_dependencies': {'python': ['requests']}, 'data': [ 'wizard/geonames_import_view.xml', ], diff --git a/base_location_geonames_import/tests/test_base_location_geonames_import.py b/base_location_geonames_import/tests/test_base_location_geonames_import.py index 303e6ad67..a81a7502b 100644 --- a/base_location_geonames_import/tests/test_base_location_geonames_import.py +++ b/base_location_geonames_import/tests/test_base_location_geonames_import.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# © 2016 Pedro M. Baeza -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +# Copyright 2016 Pedro M. Baeza +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). from odoo.tests import common @@ -10,6 +10,7 @@ class TestBaseLocationGeonamesImport(common.SavepointCase): def setUpClass(cls): super(TestBaseLocationGeonamesImport, cls).setUpClass() cls.country = cls.env.ref('base.mc') + cls.country.enforce_cities = True cls.wizard = cls.env['better.zip.geonames.import'].create({ 'country_id': cls.country.id, }) @@ -27,12 +28,25 @@ class TestBaseLocationGeonamesImport(common.SavepointCase): ('country_id', '=', self.country.id) ]) self.assertEqual(zip_count, max_import) + + # Look if there are imported cities + city_count = self.env['res.city'].search_count([ + ('country_id', '=', self.country.id) + ]) + self.assertTrue(city_count) + # Reimport again to see that there's no duplicates self.wizard.with_context(max_import=max_import).run_import() state_count2 = self.env['res.country.state'].search_count([ ('country_id', '=', self.country.id) ]) self.assertEqual(state_count, state_count2) + + city_count2 = self.env['res.city'].search_count([ + ('country_id', '=', self.country.id) + ]) + self.assertEqual(city_count, city_count2) + zip_count = self.env['res.better.zip'].search_count([ ('country_id', '=', self.country.id) ]) @@ -46,6 +60,13 @@ class TestBaseLocationGeonamesImport(common.SavepointCase): self.wizard.run_import() self.assertFalse(zip_entry.exists()) + city_entry = self.env['res.city'].create({ + 'name': 'Test city', + 'country_id': self.country.id, + }) + self.wizard.run_import() + self.assertFalse(city_entry.exists()) + def test_import_title(self): self.wizard.letter_case = 'title' self.wizard.with_context(max_import=1).run_import() @@ -54,6 +75,11 @@ class TestBaseLocationGeonamesImport(common.SavepointCase): ) self.assertEqual(zip.city, zip.city.title()) + city = self.env['res.city'].search( + [('country_id', '=', self.country.id)], limit=1 + ) + self.assertEqual(city.name, city.name.title()) + def test_import_upper(self): self.wizard.letter_case = 'upper' self.wizard.with_context(max_import=1).run_import() @@ -61,3 +87,8 @@ class TestBaseLocationGeonamesImport(common.SavepointCase): [('country_id', '=', self.country.id)], limit=1 ) self.assertEqual(zip.city, zip.city.upper()) + + city = self.env['res.city'].search( + [('country_id', '=', self.country.id)], limit=1 + ) + self.assertEqual(city.name, city.name.upper()) diff --git a/base_location_geonames_import/wizard/geonames_import.py b/base_location_geonames_import/wizard/geonames_import.py index 1d85f7652..f3f24a5c7 100644 --- a/base_location_geonames_import/wizard/geonames_import.py +++ b/base_location_geonames_import/wizard/geonames_import.py @@ -1,22 +1,21 @@ # -*- coding: utf-8 -*- -# © 2014-2016 Akretion (Alexis de Lattre ) -# © 2014 Lorenzo Battistini -# © 2016 Pedro M. Baeza -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +# Copyright 2014-2016 Akretion (Alexis de Lattre +# ) +# Copyright 2014 Lorenzo Battistini +# Copyright 2016 Pedro M. Baeza +# Copyright 2017 Eficent Business and IT Consulting Services, S.L. +# +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). from odoo import _, api, fields, models, tools from odoo.exceptions import UserError import requests import tempfile -import StringIO +import io import zipfile import os import logging - -try: - import unicodecsv -except ImportError: - unicodecsv = None +import csv logger = logging.getLogger(__name__) @@ -27,6 +26,12 @@ class BetterZipGeonamesImport(models.TransientModel): _rec_name = 'country_id' country_id = fields.Many2one('res.country', 'Country', required=True) + enforce_cities = fields.Boolean(string='Enforce Cities', + help='The city will be created as a ' + 'separate entity.', + related='country_id.enforce_cities', + readonly=True) + letter_case = fields.Selection([ ('unchanged', 'Unchanged'), ('title', 'Title Case'), @@ -46,26 +51,46 @@ class BetterZipGeonamesImport(models.TransientModel): return city @api.model - def _domain_search_better_zip(self, row, country): - return [('name', '=', row[1]), - ('city', '=', self.transform_city_name(row[2], country)), + def _domain_search_res_city(self, row, country): + return [('name', '=', self.transform_city_name(row[2], country)), ('country_id', '=', country.id)] @api.model - def _prepare_better_zip(self, row, country): + def _domain_search_better_zip(self, row, country, res_city): + domain = [('name', '=', row[1]), + ('city', '=', self.transform_city_name(row[2], country)), + ('country_id', '=', country.id)] + if res_city: + domain += [('city_id', '=', res_city.id)] + return domain + + @api.model + def _prepare_res_city(self, row, country): + state = self.select_or_create_state(row, country) + vals = { + 'name': self.transform_city_name(row[2], country), + 'state_id': state.id, + 'country_id': country.id, + } + return vals + + @api.model + def _prepare_better_zip(self, row, country, res_city): state = self.select_or_create_state(row, country) + city_name = self.transform_city_name(row[2], country) vals = { 'name': row[1], - 'city': self.transform_city_name(row[2], country), + 'city_id': res_city and res_city.id or False, + 'city': res_city and res_city.name or city_name, 'state_id': state.id, 'country_id': country.id, 'latitude': row[9], 'longitude': row[10], - } + } return vals @api.model - def create_better_zip(self, row, country): + def create_better_zip(self, row, country, res_city): if row[0] != country.code: raise UserError( _("The country code inside the file (%s) doesn't " @@ -81,16 +106,46 @@ class BetterZipGeonamesImport(models.TransientModel): if row[1] and row[2]: zip_model = self.env['res.better.zip'] zips = zip_model.search(self._domain_search_better_zip( - row, country)) + row, country, res_city)) if zips: return zips[0] else: - vals = self._prepare_better_zip(row, country) + vals = self._prepare_better_zip(row, country, res_city) if vals: + logger.debug('Creating res.better.zip %s' % vals['name']) return zip_model.create(vals) else: # pragma: no cover return False + @api.model + def create_res_city(self, row, country): + if row[0] != country.code: + raise UserError( + _("The country code inside the file (%s) doesn't " + "correspond to the selected country (%s).") + % (row[0], country.code)) + logger.debug('Processing city creation for ZIP = %s - City = %s' % + (row[1], row[2])) + if self.letter_case == 'title': + row[2] = row[2].title() + row[3] = row[3].title() + elif self.letter_case == 'upper': + row[2] = row[2].upper() + row[3] = row[3].upper() + if row[2]: + res_city_model = self.env['res.city'] + res_cities = res_city_model.search(self._domain_search_res_city( + row, country)) + if res_cities: + return res_cities[0] + else: + vals = self._prepare_res_city(row, country) + if vals: + logger.debug('Creating res.city %s' % vals['name']) + return res_city_model.create(vals) + else: # pragma: no cover + return False + @tools.ormcache('country_id', 'code') def _get_state(self, country_id, code, name): state = self.env['res.country.state'].search( @@ -117,6 +172,7 @@ class BetterZipGeonamesImport(models.TransientModel): def run_import(self): self.ensure_one() zip_model = self.env['res.better.zip'] + res_city_model = self.env['res.city'] country_code = self.country_id.code config_url = self.env['ir.config_parameter'].get_param( 'geonames.url', @@ -129,19 +185,32 @@ class BetterZipGeonamesImport(models.TransientModel): _('Got an error %d when trying to download the file %s.') % (res_request.status_code, url)) # Store current record list + + res_cities_to_delete = res_city_model zips_to_delete = zip_model.search( [('country_id', '=', self.country_id.id)]) - f_geonames = zipfile.ZipFile(StringIO.StringIO(res_request.content)) - tempdir = tempfile.mkdtemp(prefix='openerp') + if self.enforce_cities: + res_cities_to_delete = res_city_model.search( + [('country_id', '=', self.country_id.id)]) + + f_geonames = zipfile.ZipFile(io.BytesIO(res_request.content)) + tempdir = tempfile.mkdtemp(prefix='odoo') f_geonames.extract('%s.txt' % country_code, tempdir) logger.info('The geonames zipfile has been decompressed') - data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r') + data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r', + encoding='utf-8') data_file.seek(0) - logger.info('Starting to create the better zip entries') + logger.info('Starting to create the cities and/or better zip entries') max_import = self.env.context.get('max_import', 0) - reader = unicodecsv.reader(data_file, encoding='utf-8', delimiter=' ') + reader = csv.reader(data_file, delimiter=' ') for i, row in enumerate(reader): - zip_code = self.create_better_zip(row, self.country_id) + res_city = False + if self.enforce_cities: + res_city = self.create_res_city(row, self.country_id) + if res_city in res_cities_to_delete: + res_cities_to_delete -= res_city + zip_code = self.create_better_zip(row, self.country_id, + res_city) if zip_code in zips_to_delete: zips_to_delete -= zip_code if max_import and (i + 1) == max_import: @@ -151,7 +220,11 @@ class BetterZipGeonamesImport(models.TransientModel): zips_to_delete.unlink() logger.info('%d better zip entries deleted for country %s' % (len(zips_to_delete), self.country_id.name)) + if res_cities_to_delete and not max_import: + res_cities_to_delete.unlink() + logger.info('%d res.city entries deleted for country %s' % + (len(res_cities_to_delete), self.country_id.name)) logger.info( - 'The wizard to create better zip entries from geonames ' - 'has been successfully completed.') + 'The wizard to create cities and/or better zip entries from ' + 'geonames has been successfully completed.') return True diff --git a/base_location_geonames_import/wizard/geonames_import_view.xml b/base_location_geonames_import/wizard/geonames_import_view.xml index 8aab10d1b..11db59b43 100644 --- a/base_location_geonames_import/wizard/geonames_import_view.xml +++ b/base_location_geonames_import/wizard/geonames_import_view.xml @@ -13,6 +13,7 @@ +