From 336727cb2c61f5d1756fec2e6e2d3be7bac90ca0 Mon Sep 17 00:00:00 2001 From: Aitor Bouzas Date: Tue, 16 Oct 2018 16:13:56 +0200 Subject: [PATCH] [MIG] base_location_geonames_import: Migration to 12.0 Generated new readme. Adapted the module due to the refactoring of base_location. Adapted tests. --- base_location_geonames_import/README.rst | 99 ++-- base_location_geonames_import/__init__.py | 2 +- base_location_geonames_import/__manifest__.py | 16 +- .../models/__init__.py | 4 +- .../models/res_country.py | 1 - .../readme/CONFIGURE.rst | 7 + .../readme/CONTRIBUTORS.rst | 7 + .../readme/DESCRIPTION.rst | 2 + .../readme/INSTALL.rst | 4 + .../readme/USAGE.rst | 8 + .../static/description/index.html | 457 ++++++++++++++++++ .../tests/__init__.py | 1 - .../test_base_location_geonames_import.py | 51 +- .../wizard/__init__.py | 2 +- .../wizard/geonames_import.py | 317 ++++++------ .../wizard/geonames_import_view.xml | 15 +- 16 files changed, 769 insertions(+), 224 deletions(-) create mode 100644 base_location_geonames_import/readme/CONFIGURE.rst create mode 100644 base_location_geonames_import/readme/CONTRIBUTORS.rst create mode 100644 base_location_geonames_import/readme/DESCRIPTION.rst create mode 100644 base_location_geonames_import/readme/INSTALL.rst create mode 100644 base_location_geonames_import/readme/USAGE.rst create mode 100644 base_location_geonames_import/static/description/index.html diff --git a/base_location_geonames_import/README.rst b/base_location_geonames_import/README.rst index 62eac8eab..ed73d546f 100644 --- a/base_location_geonames_import/README.rst +++ b/base_location_geonames_import/README.rst @@ -1,62 +1,92 @@ -.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg - :target: https://www.gnu.org/licenses/agpl-3.0-standalone.html - :alt: License: AGPL-3 - ============================= Base Location Geonames Import ============================= -This module adds a wizard to import cities and/or better zip entries from +.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fpartner--contact-lightgray.png?logo=github + :target: https://github.com/OCA/partner-contact/tree/12.0/base_location_geonames_import + :alt: OCA/partner-contact +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/partner-contact-12-0/partner-contact-12-0-base_location_geonames_import + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png + :target: https://runbot.odoo-community.org/runbot/134/12.0 + :alt: Try me on Runbot + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This module adds a wizard to import cities and/or city zip entries from `Geonames `_ database. +**Table of contents** + +.. contents:: + :local: + Installation ============ -To install this module, you need the Python library 'requests'. +To install this module, you need the Python library 'requests':: + + pip install requests + Configuration ============= -To access the menu to import better zip entries from Geonames, -you must add yourself to the groups *Technical features* and *Sales manager*. +To access the menu to import city zip entries from Geonames +you must add yourself to the groups *Administration / Settings* or, if you have sale module +installed, *Sales / Manager* group. -If want want/need to modify the default URL +If you want/need to modify the default URL (http://download.geonames.org/export/zip/), you can set the *geonames.url* system parameter. Usage ===== -Go to *Settings > Technical > Cities/Locations Management > Import from Geonames*, +Go to *Contacts > Configuration > Localization > Import from Geonames*, and click on it to open a wizard. -When you start the wizard, it will ask you to select a country. If the -country has been set-up to require the entry of cites from a list you will be -able to indicate if you want to import the cities or zip codes. - +When you start the wizard, it will ask you to select a country. -Then, for the -selected country, it will delete all the current better zip entries, download +Then, for the selected country, it will delete all not detected entries, download the latest version of the list of cities from geonames.org and create new -better zip entries. - -.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas - :alt: Try me on Runbot - :target: https://runbot.odoo-community.org/runbot/134/11.0 +city zip entries. Bug Tracker =========== -Bugs are tracked on `GitHub Issues -`_. In case of trouble, please -check there if your issue has already been reported. If you spotted it first, -help us smashing it by providing a detailed and welcomed feedback. +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. Credits ======= +Authors +~~~~~~~ + +* Akretion +* Agile Business Group +* Tecnativa +* AdaptiveCity + Contributors ------------- +~~~~~~~~~~~~ * Alexis de Lattre * Lorenzo Battistini @@ -64,24 +94,21 @@ Contributors * Dave Lasley * Jordi Ballester * Franco Tampieri +* Aitor Bouzas -Icon ----- +Maintainers +~~~~~~~~~~~ -* http://icon-park.com/icon/location-map-pin-orange3/ -* http://commons.wikimedia.org/wiki/File:View-refresh.svg - -Maintainer ----------- +This module is maintained by the OCA. .. image:: https://odoo-community.org/logo.png :alt: Odoo Community Association :target: https://odoo-community.org -This module is maintained by the OCA. - OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -To contribute to this module, please visit https://odoo-community.org. +This module is part of the `OCA/partner-contact `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/base_location_geonames_import/__init__.py b/base_location_geonames_import/__init__.py index 35e7c9600..f987065d3 100644 --- a/base_location_geonames_import/__init__.py +++ b/base_location_geonames_import/__init__.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). from . import models from . import wizard diff --git a/base_location_geonames_import/__manifest__.py b/base_location_geonames_import/__manifest__.py index f1e2da919..441a6982f 100644 --- a/base_location_geonames_import/__manifest__.py +++ b/base_location_geonames_import/__manifest__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2014-2016 Akretion (Alexis de Lattre # ) # Copyright 2014 Lorenzo Battistini @@ -9,22 +8,23 @@ { 'name': 'Base Location Geonames Import', - 'version': '11.0.1.1.1', + 'version': '12.0.1.0.0', 'category': 'Partner Management', 'license': 'AGPL-3', - 'summary': 'Import better zip entries from Geonames', + 'summary': 'Import zip entries from Geonames', 'author': 'Akretion,' 'Agile Business Group,' - 'Antiun Ingeniería S.L.,' 'Tecnativa,' + 'AdaptiveCity,' 'Odoo Community Association (OCA)', - 'website': 'http://www.akretion.com', - 'depends': ['base_location'], - 'external_dependencies': {'python': ['requests']}, + 'website': 'https://github.com/OCA/partner-contact', + 'depends': [ + 'base_location', + ], 'data': [ 'data/res_country_data.xml', 'views/res_country_view.xml', 'wizard/geonames_import_view.xml', - ], + ], 'installable': True, } diff --git a/base_location_geonames_import/models/__init__.py b/base_location_geonames_import/models/__init__.py index 3a4c215d4..2a3ab3378 100644 --- a/base_location_geonames_import/models/__init__.py +++ b/base_location_geonames_import/models/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# Copyright 2017 Franco Tampieri, Freelancer http://franco.tampieri.info -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). from . import res_country diff --git a/base_location_geonames_import/models/res_country.py b/base_location_geonames_import/models/res_country.py index 279d2ce79..de5f1f3b5 100644 --- a/base_location_geonames_import/models/res_country.py +++ b/base_location_geonames_import/models/res_country.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2017 Franco Tampieri, Freelancer http://franco.tampieri.info # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). diff --git a/base_location_geonames_import/readme/CONFIGURE.rst b/base_location_geonames_import/readme/CONFIGURE.rst new file mode 100644 index 000000000..71a216809 --- /dev/null +++ b/base_location_geonames_import/readme/CONFIGURE.rst @@ -0,0 +1,7 @@ +To access the menu to import city zip entries from Geonames +you must add yourself to the groups *Administration / Settings* or, if you have sale module +installed, *Sales / Manager* group. + +If you want/need to modify the default URL +(http://download.geonames.org/export/zip/), you can set the *geonames.url* +system parameter. diff --git a/base_location_geonames_import/readme/CONTRIBUTORS.rst b/base_location_geonames_import/readme/CONTRIBUTORS.rst new file mode 100644 index 000000000..f4b6af656 --- /dev/null +++ b/base_location_geonames_import/readme/CONTRIBUTORS.rst @@ -0,0 +1,7 @@ +* Alexis de Lattre +* Lorenzo Battistini +* Pedro M. Baeza +* Dave Lasley +* Jordi Ballester +* Franco Tampieri +* Aitor Bouzas diff --git a/base_location_geonames_import/readme/DESCRIPTION.rst b/base_location_geonames_import/readme/DESCRIPTION.rst new file mode 100644 index 000000000..8eafc719f --- /dev/null +++ b/base_location_geonames_import/readme/DESCRIPTION.rst @@ -0,0 +1,2 @@ +This module adds a wizard to import cities and/or city zip entries from +`Geonames `_ database. diff --git a/base_location_geonames_import/readme/INSTALL.rst b/base_location_geonames_import/readme/INSTALL.rst new file mode 100644 index 000000000..87c7ef1b7 --- /dev/null +++ b/base_location_geonames_import/readme/INSTALL.rst @@ -0,0 +1,4 @@ +To install this module, you need the Python library 'requests':: + + pip install requests + diff --git a/base_location_geonames_import/readme/USAGE.rst b/base_location_geonames_import/readme/USAGE.rst new file mode 100644 index 000000000..c5c9d6bbb --- /dev/null +++ b/base_location_geonames_import/readme/USAGE.rst @@ -0,0 +1,8 @@ +Go to *Contacts > Configuration > Localization > Import from Geonames*, +and click on it to open a wizard. + +When you start the wizard, it will ask you to select a country. + +Then, for the selected country, it will delete all not detected entries, download +the latest version of the list of cities from geonames.org and create new +city zip entries. diff --git a/base_location_geonames_import/static/description/index.html b/base_location_geonames_import/static/description/index.html new file mode 100644 index 000000000..b3e51d31f --- /dev/null +++ b/base_location_geonames_import/static/description/index.html @@ -0,0 +1,457 @@ + + + + + + +Base Location Geonames Import + + + +
+

Base Location Geonames Import

+ + +

Beta License: AGPL-3 OCA/partner-contact Translate me on Weblate Try me on Runbot

+

This module adds a wizard to import cities and/or city zip entries from +Geonames database.

+

Table of contents

+ +
+

Installation

+

To install this module, you need the Python library ‘requests’:

+
+pip install requests
+
+
+
+

Configuration

+

To access the menu to import city zip entries from Geonames +you must add yourself to the groups Administration / Settings or, if you have sale module +installed, Sales / Manager group.

+

If you want/need to modify the default URL +(http://download.geonames.org/export/zip/), you can set the geonames.url +system parameter.

+
+
+

Usage

+

Go to Contacts > Configuration > Localization > Import from Geonames, +and click on it to open a wizard.

+

When you start the wizard, it will ask you to select a country.

+

Then, for the selected country, it will delete all not detected entries, download +the latest version of the list of cities from geonames.org and create new +city zip entries.

+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
  • Agile Business Group
    • +
  • Tecnativa
    • +
  • AdaptiveCity
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+Odoo Community Association +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/partner-contact project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/base_location_geonames_import/tests/__init__.py b/base_location_geonames_import/tests/__init__.py index b16fdcb7f..fef55430b 100644 --- a/base_location_geonames_import/tests/__init__.py +++ b/base_location_geonames_import/tests/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). from . import test_base_location_geonames_import diff --git a/base_location_geonames_import/tests/test_base_location_geonames_import.py b/base_location_geonames_import/tests/test_base_location_geonames_import.py index a81a7502b..d441b2e95 100644 --- a/base_location_geonames_import/tests/test_base_location_geonames_import.py +++ b/base_location_geonames_import/tests/test_base_location_geonames_import.py @@ -1,19 +1,30 @@ -# -*- coding: utf-8 -*- # Copyright 2016 Pedro M. Baeza # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). from odoo.tests import common +from odoo.exceptions import UserError class TestBaseLocationGeonamesImport(common.SavepointCase): + @classmethod def setUpClass(cls): - super(TestBaseLocationGeonamesImport, cls).setUpClass() + super().setUpClass() cls.country = cls.env.ref('base.mc') - cls.country.enforce_cities = True - cls.wizard = cls.env['better.zip.geonames.import'].create({ + cls.city = cls.env['res.city'].create({ + 'name': 'Test city', + 'country_id': cls.country.id, + }) + cls.wizard = cls.env['city.zip.geonames.import'].create({ 'country_id': cls.country.id, }) + cls.wrong_country = cls.env['res.country'].create({ + 'name': 'Wrong country', + 'code': 'ZZYYXX', + }) + cls.wrong_wizard = cls.env['city.zip.geonames.import'].create({ + 'country_id': cls.wrong_country.id, + }) def test_import_country(self): max_import = 10 @@ -24,8 +35,8 @@ class TestBaseLocationGeonamesImport(common.SavepointCase): ]) self.assertTrue(state_count) # Look if there are imported zips - zip_count = self.env['res.better.zip'].search_count([ - ('country_id', '=', self.country.id) + zip_count = self.env['res.city.zip'].search_count([ + ('city_id.country_id', '=', self.country.id) ]) self.assertEqual(zip_count, max_import) @@ -47,15 +58,15 @@ class TestBaseLocationGeonamesImport(common.SavepointCase): ]) self.assertEqual(city_count, city_count2) - zip_count = self.env['res.better.zip'].search_count([ - ('country_id', '=', self.country.id) + zip_count = self.env['res.city.zip'].search_count([ + ('city_id.country_id', '=', self.country.id) ]) self.assertEqual(zip_count, max_import) def test_delete_old_entries(self): - zip_entry = self.env['res.better.zip'].create({ - 'city': 'Test city', - 'country_id': self.country.id, + zip_entry = self.env['res.city.zip'].create({ + 'name': 'Brussels', + 'city_id': self.city.id, }) self.wizard.run_import() self.assertFalse(zip_entry.exists()) @@ -70,10 +81,10 @@ class TestBaseLocationGeonamesImport(common.SavepointCase): def test_import_title(self): self.wizard.letter_case = 'title' self.wizard.with_context(max_import=1).run_import() - zip = self.env['res.better.zip'].search( - [('country_id', '=', self.country.id)], limit=1 + zip = self.env['res.city.zip'].search( + [('city_id.country_id', '=', self.country.id)], limit=1 ) - self.assertEqual(zip.city, zip.city.title()) + self.assertEqual(zip.city_id.name, zip.city_id.name.title()) city = self.env['res.city'].search( [('country_id', '=', self.country.id)], limit=1 @@ -83,12 +94,18 @@ class TestBaseLocationGeonamesImport(common.SavepointCase): def test_import_upper(self): self.wizard.letter_case = 'upper' self.wizard.with_context(max_import=1).run_import() - zip = self.env['res.better.zip'].search( - [('country_id', '=', self.country.id)], limit=1 + zip = self.env['res.city.zip'].search( + [('city_id.country_id', '=', self.country.id)], limit=1 ) - self.assertEqual(zip.city, zip.city.upper()) + self.assertEqual(zip.city_id.name, zip.city_id.name.upper()) city = self.env['res.city'].search( [('country_id', '=', self.country.id)], limit=1 ) self.assertEqual(city.name, city.name.upper()) + + def test_download_error(self): + """Check that we get an error when trying to download + with a wrong country code""" + with self.assertRaises(UserError): + self.wrong_wizard.run_import() diff --git a/base_location_geonames_import/wizard/__init__.py b/base_location_geonames_import/wizard/__init__.py index 856fa2d2d..b31600194 100644 --- a/base_location_geonames_import/wizard/__init__.py +++ b/base_location_geonames_import/wizard/__init__.py @@ -1,3 +1,3 @@ -# -*- coding: utf-8 -*- +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). from . import geonames_import diff --git a/base_location_geonames_import/wizard/geonames_import.py b/base_location_geonames_import/wizard/geonames_import.py index 163491f27..03e7e5233 100644 --- a/base_location_geonames_import/wizard/geonames_import.py +++ b/base_location_geonames_import/wizard/geonames_import.py @@ -1,13 +1,13 @@ -# -*- coding: utf-8 -*- # Copyright 2014-2016 Akretion (Alexis de Lattre # ) # Copyright 2014 Lorenzo Battistini # Copyright 2016 Pedro M. Baeza # Copyright 2017 Eficent Business and IT Consulting Services, S.L. # +# Copyright 2018 Aitor Bouzas # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). -from odoo import _, api, fields, models, tools +from odoo import _, api, fields, models from odoo.exceptions import UserError import requests import tempfile @@ -20,26 +20,26 @@ import csv logger = logging.getLogger(__name__) -class BetterZipGeonamesImport(models.TransientModel): - _name = 'better.zip.geonames.import' - _description = 'Import Better Zip from Geonames' +class CityZipGeonamesImport(models.TransientModel): + _name = 'city.zip.geonames.import' + _description = 'Import City Zips from Geonames' _rec_name = 'country_id' country_id = fields.Many2one('res.country', 'Country', required=True) - enforce_cities = fields.Boolean(string='Enforce Cities', - help='The city will be created as a ' - 'separate entity.', - related='country_id.enforce_cities', - readonly=True) + code_row_index = fields.Integer( + related='country_id.geonames_state_code_column', + readonly=True) + name_row_index = fields.Integer( + related='country_id.geonames_state_name_column') letter_case = fields.Selection([ ('unchanged', 'Unchanged'), ('title', 'Title Case'), ('upper', 'Upper Case'), - ], string='Letter Case', default='unchanged', + ], string='Letter Case', default='unchanged', help="Converts retreived city and state names to Title Case " - "(upper case on each first letter of a word) or Upper Case " - "(all letters upper case).") + "(upper case on each first letter of a word) or Upper Case " + "(all letters upper case).") @api.model def transform_city_name(self, city, country): @@ -48,7 +48,12 @@ class BetterZipGeonamesImport(models.TransientModel): :param country: Country record :return: Transformed city name """ - return city + res = city + if self.letter_case == 'title': + res = city.title() + elif self.letter_case == 'upper': + res = city.upper() + return res @api.model def _domain_search_res_city(self, row, country): @@ -56,127 +61,59 @@ class BetterZipGeonamesImport(models.TransientModel): ('country_id', '=', country.id)] @api.model - def _domain_search_better_zip(self, row, country, res_city): - domain = [('name', '=', row[1]), - ('city', '=', self.transform_city_name(row[2], country)), - ('country_id', '=', country.id)] + def _domain_search_city_zip(self, row, res_city): + domain = [('name', '=', row[1])] if res_city: domain += [('city_id', '=', res_city.id)] return domain @api.model - def _prepare_res_city(self, row, country): - state = self.select_or_create_state(row, country) + def select_state(self, row, country): + code = row[self.code_row_index or 4] + return self.env['res.country.state'].search( + [('country_id', '=', country.id), + ('code', '=', code)], limit=1, + ) + + @api.model + def select_city(self, row, country): + res_city_model = self.env['res.city'] + return res_city_model.search(self._domain_search_res_city( + row, country), limit=1) + + @api.model + def select_zip(self, row, country): + city = self.select_city(row, country) + return self.env['res.city.zip'].search(self._domain_search_city_zip( + row, city)) + + @api.model + def prepare_state(self, row, country): + return { + 'name': row[self.name_row_index or 3], + 'code': row[self.code_row_index or 4], + 'country_id': country.id, + } + + @api.model + def prepare_city(self, row, country, state_id): vals = { 'name': self.transform_city_name(row[2], country), - 'state_id': state.id, + 'state_id': state_id, 'country_id': country.id, } return vals @api.model - def _prepare_better_zip(self, row, country, res_city): - state = self.select_or_create_state(row, country) - city_name = self.transform_city_name(row[2], country) + def prepare_zip(self, row, city_id): vals = { 'name': row[1], - 'city_id': res_city and res_city.id or False, - 'city': res_city and res_city.name or city_name, - 'state_id': state.id, - 'country_id': country.id, - 'latitude': row[9], - 'longitude': row[10], + 'city_id': city_id, } return vals @api.model - def create_better_zip(self, row, country, res_city): - if row[0] != country.code: - raise UserError( - _("The country code inside the file (%s) doesn't " - "correspond to the selected country (%s).") - % (row[0], country.code)) - logger.debug('ZIP = %s - City = %s' % (row[1], row[2])) - if self.letter_case == 'title': - row[2] = row[2].title() - row[3] = row[3].title() - elif self.letter_case == 'upper': - row[2] = row[2].upper() - row[3] = row[3].upper() - if row[1] and row[2]: - zip_model = self.env['res.better.zip'] - zips = zip_model.search(self._domain_search_better_zip( - row, country, res_city)) - if zips: - return zips[0] - else: - vals = self._prepare_better_zip(row, country, res_city) - if vals: - logger.debug('Creating res.better.zip %s' % vals['name']) - return zip_model.create(vals) - else: # pragma: no cover - return False - - @api.model - def create_res_city(self, row, country): - if row[0] != country.code: - raise UserError( - _("The country code inside the file (%s) doesn't " - "correspond to the selected country (%s).") - % (row[0], country.code)) - logger.debug('Processing city creation for ZIP = %s - City = %s' % - (row[1], row[2])) - if self.letter_case == 'title': - row[2] = row[2].title() - row[3] = row[3].title() - elif self.letter_case == 'upper': - row[2] = row[2].upper() - row[3] = row[3].upper() - if row[2]: - res_city_model = self.env['res.city'] - res_cities = res_city_model.search(self._domain_search_res_city( - row, country)) - if res_cities: - return res_cities[0] - else: - vals = self._prepare_res_city(row, country) - if vals: - logger.debug('Creating res.city %s' % vals['name']) - return res_city_model.create(vals) - else: # pragma: no cover - return False - - @tools.ormcache('country_id', 'code') - def _get_state(self, country_id, code, name): - state = self.env['res.country.state'].search( - [('country_id', '=', country_id), - ('code', '=', code)], limit=1, - ) - if state: # pragma: no cover - return state - else: - return self.env['res.country.state'].create({ - 'name': name, - 'code': code, - 'country_id': country_id, - }) - - @api.model - def select_or_create_state( - self, row, country, code_row_index=4, name_row_index=3): - if country.geonames_state_code_column: - code_row_index = country.geonames_state_code_column - if country.geonames_state_name_column: - name_row_index = country.geonames_state_name_column - return self._get_state( - country.id, row[code_row_index], row[name_row_index], - ) - - @api.multi - def run_import(self): - self.ensure_one() - zip_model = self.env['res.better.zip'] - res_city_model = self.env['res.city'] + def get_and_parse_csv(self): country_code = self.country_id.code config_url = self.env['ir.config_parameter'].get_param( 'geonames.url', @@ -188,47 +125,131 @@ class BetterZipGeonamesImport(models.TransientModel): raise UserError( _('Got an error %d when trying to download the file %s.') % (res_request.status_code, url)) - # Store current record list - - res_cities_to_delete = res_city_model - zips_to_delete = zip_model.search( - [('country_id', '=', self.country_id.id)]) - if self.enforce_cities: - res_cities_to_delete = res_city_model.search( - [('country_id', '=', self.country_id.id)]) f_geonames = zipfile.ZipFile(io.BytesIO(res_request.content)) tempdir = tempfile.mkdtemp(prefix='odoo') f_geonames.extract('%s.txt' % country_code, tempdir) - logger.info('The geonames zipfile has been decompressed') + data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r', encoding='utf-8') data_file.seek(0) - logger.info('Starting to create the cities and/or better zip entries') - max_import = self.env.context.get('max_import', 0) reader = csv.reader(data_file, delimiter=' ') - for i, row in enumerate(reader): - res_city = False - if self.enforce_cities: - res_city = self.create_res_city(row, self.country_id) - if res_city in res_cities_to_delete: - res_cities_to_delete -= res_city - zip_code = self.create_better_zip(row, self.country_id, - res_city) - if zip_code in zips_to_delete: - zips_to_delete -= zip_code - if max_import and (i + 1) == max_import: - break + parsed_csv = [row for i, row in enumerate(reader)] data_file.close() - if zips_to_delete and not max_import: - zips_to_delete.unlink() - logger.info('%d better zip entries deleted for country %s' % - (len(zips_to_delete), self.country_id.name)) - if res_cities_to_delete and not max_import: - res_cities_to_delete.unlink() + logger.info('The geonames zipfile has been decompressed') + return parsed_csv + + def _create_states(self, parsed_csv, search_states, max_import): + # States + state_vals_list = [] + state_dict = {} + for i, row in enumerate(parsed_csv): + if max_import and i == max_import: + break + state = self.select_state( + row, self.country_id) if search_states else False + if not state: + state_vals = self.prepare_state(row, self.country_id) + if state_vals not in state_vals_list: + state_vals_list.append(state_vals) + else: + state_dict[state.code] = state.id + + created_states = self.env['res.country.state'].create(state_vals_list) + for i, vals in enumerate(state_vals_list): + state_dict[vals['code']] = created_states[i].id + return state_dict + + def _create_cities(self, parsed_csv, + search_cities, max_import, state_dict): + # Cities + city_vals_list = [] + city_dict = {} + for i, row in enumerate(parsed_csv): + if max_import and i == max_import: + break + city = self.select_city( + row, self.country_id) if search_cities else False + if not city: + state_id = state_dict[ + row[self.code_row_index or 4]] + city_vals = self.prepare_city( + row, self.country_id, state_id) + if city_vals not in city_vals_list: + city_vals_list.append(city_vals) + else: + city_dict[city.name] = city.id + + created_cities = self.env['res.city'].create(city_vals_list) + for i, vals in enumerate(city_vals_list): + city_dict[vals['name']] = created_cities[i].id + return city_dict + + @api.multi + def run_import(self): + self.ensure_one() + state_model = self.env['res.country.state'] + zip_model = self.env['res.city.zip'] + res_city_model = self.env['res.city'] + + # Store current record list + current_zips = zip_model.search( + [('city_id.country_id', '=', self.country_id.id)]) + search_zips = True and len(current_zips) > 0 or False + current_cities = res_city_model.search( + [('country_id', '=', self.country_id.id)]) + search_cities = True and len(current_cities) > 0 or False + current_states = state_model.search( + [('country_id', '=', self.country_id.id)]) + search_states = True and len(current_states) > 0 or False + + parsed_csv = self.get_and_parse_csv() + max_import = self.env.context.get('max_import', 0) + logger.info('Starting to create the cities and/or city zip entries') + + state_dict = self._create_states(parsed_csv, + search_states, max_import) + city_dict = self._create_cities(parsed_csv, + search_cities, max_import, state_dict) + + # Zips + zip_vals_list = [] + for i, row in enumerate(parsed_csv): + if max_import and i == max_import: + break + # Don't search if there aren't any records + zip = False + if search_zips: + zip = self.select_zip(row, self.country_id) + if not zip: + city_id = city_dict[ + self.transform_city_name(row[2], self.country_id)] + zip_vals = self.prepare_zip(row, city_id) + if zip_vals not in zip_vals_list: + zip_vals_list.append(zip_vals) + + delete_zips = self.env['res.city.zip'].create(zip_vals_list) + current_zips -= delete_zips + + if not max_import: + current_zips.unlink() + logger.info('%d city zip entries deleted for country %s' % + (len(current_zips), self.country_id.name)) + + # Since we wrapped the entire cities + # creation in a function we need + # to perform a search with city_dict in + # order to know which are the new ones so + # we can delete the old ones + created_cities = res_city_model.search( + [('country_id', '=', self.country_id.id), + ('id', 'in', [value for key, value in city_dict.items()])] + ) + current_cities -= created_cities + current_cities.unlink() logger.info('%d res.city entries deleted for country %s' % - (len(res_cities_to_delete), self.country_id.name)) + (len(current_cities), self.country_id.name)) logger.info( - 'The wizard to create cities and/or better zip entries from ' + 'The wizard to create cities and/or city zip entries from ' 'geonames has been successfully completed.') return True diff --git a/base_location_geonames_import/wizard/geonames_import_view.xml b/base_location_geonames_import/wizard/geonames_import_view.xml index 11db59b43..cfe014dff 100644 --- a/base_location_geonames_import/wizard/geonames_import_view.xml +++ b/base_location_geonames_import/wizard/geonames_import_view.xml @@ -1,9 +1,9 @@ - + Import from Geonames form view - better.zip.geonames.import + city.zip.geonames.import
@@ -13,7 +13,6 @@ -