Browse Source

migration to 11.0

pull/643/head
Jordi Ballester 7 years ago
committed by Pedro M. Baeza
parent
commit
66b9d77a4f
  1. 23
      base_location_geonames_import/README.rst
  2. 15
      base_location_geonames_import/__manifest__.py
  3. 35
      base_location_geonames_import/tests/test_base_location_geonames_import.py
  4. 123
      base_location_geonames_import/wizard/geonames_import.py
  5. 1
      base_location_geonames_import/wizard/geonames_import_view.xml

23
base_location_geonames_import/README.rst

@ -1,23 +1,24 @@
.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
:target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
:target: https://www.gnu.org/licenses/agpl-3.0-standalone.html
:alt: License: AGPL-3
=============================
Base Location Geonames Import
=============================
This module adds a wizard to import better zip entries from `Geonames <http://www.geonames.org/>`_ database.
This module adds a wizard to import cities and/or better zip entries from
`Geonames <http://www.geonames.org/>`_ database.
Installation
============
To install this module, you need these Python libraries: requests and
unicodecsv.
To install this module, you need the Python library 'requests'.
Configuration
=============
To access the menu to import better zip entries from Geonames, you must add yourself to the groups *Technical features* and *Sales manager*.
To access the menu to import better zip entries from Geonames,
you must add yourself to the groups *Technical features* and *Sales manager*.
If want want/need to modify the default URL
(http://download.geonames.org/export/zip/), you can set the *geonames.url*
@ -29,14 +30,19 @@ Usage
Go to *Settings > Technical > Cities/Locations Management > Import from Geonames*,
and click on it to open a wizard.
When you start the wizard, it will ask you to select a country. Then, for the
When you start the wizard, it will ask you to select a country. If the
country has been set-up to require the entry of cites from a list you will be
able to indicate if you want to import the cities or zip codes.
Then, for the
selected country, it will delete all the current better zip entries, download
the latest version of the list of cities from geonames.org and create new
better zip entries.
.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas
:alt: Try me on Runbot
:target: https://runbot.odoo-community.org/runbot/134/10.0
:target: https://runbot.odoo-community.org/runbot/134/11.0
Bug Tracker
===========
@ -54,8 +60,9 @@ Contributors
* Alexis de Lattre <alexis.delattre@akretion.com>
* Lorenzo Battistini <lorenzo.battistini@agilebg.com>
* Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
* Pedro M. Baeza <pedro.baeza@tecnativa.com>
* Dave Lasley <dave@laslabs.com>
* Jordi Ballester <jordi.ballester@eficent.com>
Icon
----

15
base_location_geonames_import/__manifest__.py

@ -1,12 +1,15 @@
# -*- coding: utf-8 -*-
# © 2014-2016 Akretion (Alexis de Lattre <alexis.delattre@akretion.com>)
# © 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
# © 2016 Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
# Copyright 2014-2016 Akretion (Alexis de Lattre
# <alexis.delattre@akretion.com>)
# Copyright 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
# Copyright 2016 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# Copyright 2017 Eficent Business and IT Consulting Services, S.L.
# <contact@eficent.com>
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
{
'name': 'Base Location Geonames Import',
'version': '10.0.1.0.1',
'version': '11.0.1.0.1',
'category': 'Partner Management',
'license': 'AGPL-3',
'summary': 'Import better zip entries from Geonames',
@ -17,7 +20,7 @@
'Odoo Community Association (OCA)',
'website': 'http://www.akretion.com',
'depends': ['base_location'],
'external_dependencies': {'python': ['requests', 'unicodecsv']},
'external_dependencies': {'python': ['requests']},
'data': [
'wizard/geonames_import_view.xml',
],

35
base_location_geonames_import/tests/test_base_location_geonames_import.py

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# © 2016 Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
# Copyright 2016 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
from odoo.tests import common
@ -10,6 +10,7 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
def setUpClass(cls):
super(TestBaseLocationGeonamesImport, cls).setUpClass()
cls.country = cls.env.ref('base.mc')
cls.country.enforce_cities = True
cls.wizard = cls.env['better.zip.geonames.import'].create({
'country_id': cls.country.id,
})
@ -27,12 +28,25 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
('country_id', '=', self.country.id)
])
self.assertEqual(zip_count, max_import)
# Look if there are imported cities
city_count = self.env['res.city'].search_count([
('country_id', '=', self.country.id)
])
self.assertTrue(city_count)
# Reimport again to see that there's no duplicates
self.wizard.with_context(max_import=max_import).run_import()
state_count2 = self.env['res.country.state'].search_count([
('country_id', '=', self.country.id)
])
self.assertEqual(state_count, state_count2)
city_count2 = self.env['res.city'].search_count([
('country_id', '=', self.country.id)
])
self.assertEqual(city_count, city_count2)
zip_count = self.env['res.better.zip'].search_count([
('country_id', '=', self.country.id)
])
@ -46,6 +60,13 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
self.wizard.run_import()
self.assertFalse(zip_entry.exists())
city_entry = self.env['res.city'].create({
'name': 'Test city',
'country_id': self.country.id,
})
self.wizard.run_import()
self.assertFalse(city_entry.exists())
def test_import_title(self):
self.wizard.letter_case = 'title'
self.wizard.with_context(max_import=1).run_import()
@ -54,6 +75,11 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
)
self.assertEqual(zip.city, zip.city.title())
city = self.env['res.city'].search(
[('country_id', '=', self.country.id)], limit=1
)
self.assertEqual(city.name, city.name.title())
def test_import_upper(self):
self.wizard.letter_case = 'upper'
self.wizard.with_context(max_import=1).run_import()
@ -61,3 +87,8 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
[('country_id', '=', self.country.id)], limit=1
)
self.assertEqual(zip.city, zip.city.upper())
city = self.env['res.city'].search(
[('country_id', '=', self.country.id)], limit=1
)
self.assertEqual(city.name, city.name.upper())

123
base_location_geonames_import/wizard/geonames_import.py

@ -1,22 +1,21 @@
# -*- coding: utf-8 -*-
# © 2014-2016 Akretion (Alexis de Lattre <alexis.delattre@akretion.com>)
# © 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
# © 2016 Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
# Copyright 2014-2016 Akretion (Alexis de Lattre
# <alexis.delattre@akretion.com>)
# Copyright 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
# Copyright 2016 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# Copyright 2017 Eficent Business and IT Consulting Services, S.L.
# <contact@eficent.com>
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
from odoo import _, api, fields, models, tools
from odoo.exceptions import UserError
import requests
import tempfile
import StringIO
import io
import zipfile
import os
import logging
try:
import unicodecsv
except ImportError:
unicodecsv = None
import csv
logger = logging.getLogger(__name__)
@ -27,6 +26,12 @@ class BetterZipGeonamesImport(models.TransientModel):
_rec_name = 'country_id'
country_id = fields.Many2one('res.country', 'Country', required=True)
enforce_cities = fields.Boolean(string='Enforce Cities',
help='The city will be created as a '
'separate entity.',
related='country_id.enforce_cities',
readonly=True)
letter_case = fields.Selection([
('unchanged', 'Unchanged'),
('title', 'Title Case'),
@ -46,17 +51,37 @@ class BetterZipGeonamesImport(models.TransientModel):
return city
@api.model
def _domain_search_better_zip(self, row, country):
return [('name', '=', row[1]),
def _domain_search_res_city(self, row, country):
return [('name', '=', self.transform_city_name(row[2], country)),
('country_id', '=', country.id)]
@api.model
def _domain_search_better_zip(self, row, country, res_city):
domain = [('name', '=', row[1]),
('city', '=', self.transform_city_name(row[2], country)),
('country_id', '=', country.id)]
if res_city:
domain += [('city_id', '=', res_city.id)]
return domain
@api.model
def _prepare_res_city(self, row, country):
state = self.select_or_create_state(row, country)
vals = {
'name': self.transform_city_name(row[2], country),
'state_id': state.id,
'country_id': country.id,
}
return vals
@api.model
def _prepare_better_zip(self, row, country):
def _prepare_better_zip(self, row, country, res_city):
state = self.select_or_create_state(row, country)
city_name = self.transform_city_name(row[2], country)
vals = {
'name': row[1],
'city': self.transform_city_name(row[2], country),
'city_id': res_city and res_city.id or False,
'city': res_city and res_city.name or city_name,
'state_id': state.id,
'country_id': country.id,
'latitude': row[9],
@ -65,7 +90,7 @@ class BetterZipGeonamesImport(models.TransientModel):
return vals
@api.model
def create_better_zip(self, row, country):
def create_better_zip(self, row, country, res_city):
if row[0] != country.code:
raise UserError(
_("The country code inside the file (%s) doesn't "
@ -81,16 +106,46 @@ class BetterZipGeonamesImport(models.TransientModel):
if row[1] and row[2]:
zip_model = self.env['res.better.zip']
zips = zip_model.search(self._domain_search_better_zip(
row, country))
row, country, res_city))
if zips:
return zips[0]
else:
vals = self._prepare_better_zip(row, country)
vals = self._prepare_better_zip(row, country, res_city)
if vals:
logger.debug('Creating res.better.zip %s' % vals['name'])
return zip_model.create(vals)
else: # pragma: no cover
return False
@api.model
def create_res_city(self, row, country):
if row[0] != country.code:
raise UserError(
_("The country code inside the file (%s) doesn't "
"correspond to the selected country (%s).")
% (row[0], country.code))
logger.debug('Processing city creation for ZIP = %s - City = %s' %
(row[1], row[2]))
if self.letter_case == 'title':
row[2] = row[2].title()
row[3] = row[3].title()
elif self.letter_case == 'upper':
row[2] = row[2].upper()
row[3] = row[3].upper()
if row[2]:
res_city_model = self.env['res.city']
res_cities = res_city_model.search(self._domain_search_res_city(
row, country))
if res_cities:
return res_cities[0]
else:
vals = self._prepare_res_city(row, country)
if vals:
logger.debug('Creating res.city %s' % vals['name'])
return res_city_model.create(vals)
else: # pragma: no cover
return False
@tools.ormcache('country_id', 'code')
def _get_state(self, country_id, code, name):
state = self.env['res.country.state'].search(
@ -117,6 +172,7 @@ class BetterZipGeonamesImport(models.TransientModel):
def run_import(self):
self.ensure_one()
zip_model = self.env['res.better.zip']
res_city_model = self.env['res.city']
country_code = self.country_id.code
config_url = self.env['ir.config_parameter'].get_param(
'geonames.url',
@ -129,19 +185,32 @@ class BetterZipGeonamesImport(models.TransientModel):
_('Got an error %d when trying to download the file %s.')
% (res_request.status_code, url))
# Store current record list
res_cities_to_delete = res_city_model
zips_to_delete = zip_model.search(
[('country_id', '=', self.country_id.id)])
f_geonames = zipfile.ZipFile(StringIO.StringIO(res_request.content))
tempdir = tempfile.mkdtemp(prefix='openerp')
if self.enforce_cities:
res_cities_to_delete = res_city_model.search(
[('country_id', '=', self.country_id.id)])
f_geonames = zipfile.ZipFile(io.BytesIO(res_request.content))
tempdir = tempfile.mkdtemp(prefix='odoo')
f_geonames.extract('%s.txt' % country_code, tempdir)
logger.info('The geonames zipfile has been decompressed')
data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r')
data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r',
encoding='utf-8')
data_file.seek(0)
logger.info('Starting to create the better zip entries')
logger.info('Starting to create the cities and/or better zip entries')
max_import = self.env.context.get('max_import', 0)
reader = unicodecsv.reader(data_file, encoding='utf-8', delimiter=' ')
reader = csv.reader(data_file, delimiter=' ')
for i, row in enumerate(reader):
zip_code = self.create_better_zip(row, self.country_id)
res_city = False
if self.enforce_cities:
res_city = self.create_res_city(row, self.country_id)
if res_city in res_cities_to_delete:
res_cities_to_delete -= res_city
zip_code = self.create_better_zip(row, self.country_id,
res_city)
if zip_code in zips_to_delete:
zips_to_delete -= zip_code
if max_import and (i + 1) == max_import:
@ -151,7 +220,11 @@ class BetterZipGeonamesImport(models.TransientModel):
zips_to_delete.unlink()
logger.info('%d better zip entries deleted for country %s' %
(len(zips_to_delete), self.country_id.name))
if res_cities_to_delete and not max_import:
res_cities_to_delete.unlink()
logger.info('%d res.city entries deleted for country %s' %
(len(res_cities_to_delete), self.country_id.name))
logger.info(
'The wizard to create better zip entries from geonames '
'has been successfully completed.')
'The wizard to create cities and/or better zip entries from '
'geonames has been successfully completed.')
return True

1
base_location_geonames_import/wizard/geonames_import_view.xml

@ -13,6 +13,7 @@
</group>
<group name="main">
<field name="country_id"/>
<field name="enforce_cities"/>
<field name="letter_case"/>
</group>
<footer>

Loading…
Cancel
Save