Browse Source

migration to 11.0

pull/643/head
Jordi Ballester 7 years ago
committed by Pedro M. Baeza
parent
commit
66b9d77a4f
  1. 23
      base_location_geonames_import/README.rst
  2. 15
      base_location_geonames_import/__manifest__.py
  3. 35
      base_location_geonames_import/tests/test_base_location_geonames_import.py
  4. 127
      base_location_geonames_import/wizard/geonames_import.py
  5. 1
      base_location_geonames_import/wizard/geonames_import_view.xml

23
base_location_geonames_import/README.rst

@ -1,23 +1,24 @@
.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg .. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
:target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
:target: https://www.gnu.org/licenses/agpl-3.0-standalone.html
:alt: License: AGPL-3 :alt: License: AGPL-3
============================= =============================
Base Location Geonames Import Base Location Geonames Import
============================= =============================
This module adds a wizard to import better zip entries from `Geonames <http://www.geonames.org/>`_ database.
This module adds a wizard to import cities and/or better zip entries from
`Geonames <http://www.geonames.org/>`_ database.
Installation Installation
============ ============
To install this module, you need these Python libraries: requests and
unicodecsv.
To install this module, you need the Python library 'requests'.
Configuration Configuration
============= =============
To access the menu to import better zip entries from Geonames, you must add yourself to the groups *Technical features* and *Sales manager*.
To access the menu to import better zip entries from Geonames,
you must add yourself to the groups *Technical features* and *Sales manager*.
If want want/need to modify the default URL If want want/need to modify the default URL
(http://download.geonames.org/export/zip/), you can set the *geonames.url* (http://download.geonames.org/export/zip/), you can set the *geonames.url*
@ -29,14 +30,19 @@ Usage
Go to *Settings > Technical > Cities/Locations Management > Import from Geonames*, Go to *Settings > Technical > Cities/Locations Management > Import from Geonames*,
and click on it to open a wizard. and click on it to open a wizard.
When you start the wizard, it will ask you to select a country. Then, for the
When you start the wizard, it will ask you to select a country. If the
country has been set-up to require the entry of cites from a list you will be
able to indicate if you want to import the cities or zip codes.
Then, for the
selected country, it will delete all the current better zip entries, download selected country, it will delete all the current better zip entries, download
the latest version of the list of cities from geonames.org and create new the latest version of the list of cities from geonames.org and create new
better zip entries. better zip entries.
.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas .. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas
:alt: Try me on Runbot :alt: Try me on Runbot
:target: https://runbot.odoo-community.org/runbot/134/10.0
:target: https://runbot.odoo-community.org/runbot/134/11.0
Bug Tracker Bug Tracker
=========== ===========
@ -54,8 +60,9 @@ Contributors
* Alexis de Lattre <alexis.delattre@akretion.com> * Alexis de Lattre <alexis.delattre@akretion.com>
* Lorenzo Battistini <lorenzo.battistini@agilebg.com> * Lorenzo Battistini <lorenzo.battistini@agilebg.com>
* Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
* Pedro M. Baeza <pedro.baeza@tecnativa.com>
* Dave Lasley <dave@laslabs.com> * Dave Lasley <dave@laslabs.com>
* Jordi Ballester <jordi.ballester@eficent.com>
Icon Icon
---- ----

15
base_location_geonames_import/__manifest__.py

@ -1,12 +1,15 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# © 2014-2016 Akretion (Alexis de Lattre <alexis.delattre@akretion.com>)
# © 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
# © 2016 Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
# Copyright 2014-2016 Akretion (Alexis de Lattre
# <alexis.delattre@akretion.com>)
# Copyright 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
# Copyright 2016 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# Copyright 2017 Eficent Business and IT Consulting Services, S.L.
# <contact@eficent.com>
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
{ {
'name': 'Base Location Geonames Import', 'name': 'Base Location Geonames Import',
'version': '10.0.1.0.1',
'version': '11.0.1.0.1',
'category': 'Partner Management', 'category': 'Partner Management',
'license': 'AGPL-3', 'license': 'AGPL-3',
'summary': 'Import better zip entries from Geonames', 'summary': 'Import better zip entries from Geonames',
@ -17,7 +20,7 @@
'Odoo Community Association (OCA)', 'Odoo Community Association (OCA)',
'website': 'http://www.akretion.com', 'website': 'http://www.akretion.com',
'depends': ['base_location'], 'depends': ['base_location'],
'external_dependencies': {'python': ['requests', 'unicodecsv']},
'external_dependencies': {'python': ['requests']},
'data': [ 'data': [
'wizard/geonames_import_view.xml', 'wizard/geonames_import_view.xml',
], ],

35
base_location_geonames_import/tests/test_base_location_geonames_import.py

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# © 2016 Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
# Copyright 2016 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
from odoo.tests import common from odoo.tests import common
@ -10,6 +10,7 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
def setUpClass(cls): def setUpClass(cls):
super(TestBaseLocationGeonamesImport, cls).setUpClass() super(TestBaseLocationGeonamesImport, cls).setUpClass()
cls.country = cls.env.ref('base.mc') cls.country = cls.env.ref('base.mc')
cls.country.enforce_cities = True
cls.wizard = cls.env['better.zip.geonames.import'].create({ cls.wizard = cls.env['better.zip.geonames.import'].create({
'country_id': cls.country.id, 'country_id': cls.country.id,
}) })
@ -27,12 +28,25 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
('country_id', '=', self.country.id) ('country_id', '=', self.country.id)
]) ])
self.assertEqual(zip_count, max_import) self.assertEqual(zip_count, max_import)
# Look if there are imported cities
city_count = self.env['res.city'].search_count([
('country_id', '=', self.country.id)
])
self.assertTrue(city_count)
# Reimport again to see that there's no duplicates # Reimport again to see that there's no duplicates
self.wizard.with_context(max_import=max_import).run_import() self.wizard.with_context(max_import=max_import).run_import()
state_count2 = self.env['res.country.state'].search_count([ state_count2 = self.env['res.country.state'].search_count([
('country_id', '=', self.country.id) ('country_id', '=', self.country.id)
]) ])
self.assertEqual(state_count, state_count2) self.assertEqual(state_count, state_count2)
city_count2 = self.env['res.city'].search_count([
('country_id', '=', self.country.id)
])
self.assertEqual(city_count, city_count2)
zip_count = self.env['res.better.zip'].search_count([ zip_count = self.env['res.better.zip'].search_count([
('country_id', '=', self.country.id) ('country_id', '=', self.country.id)
]) ])
@ -46,6 +60,13 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
self.wizard.run_import() self.wizard.run_import()
self.assertFalse(zip_entry.exists()) self.assertFalse(zip_entry.exists())
city_entry = self.env['res.city'].create({
'name': 'Test city',
'country_id': self.country.id,
})
self.wizard.run_import()
self.assertFalse(city_entry.exists())
def test_import_title(self): def test_import_title(self):
self.wizard.letter_case = 'title' self.wizard.letter_case = 'title'
self.wizard.with_context(max_import=1).run_import() self.wizard.with_context(max_import=1).run_import()
@ -54,6 +75,11 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
) )
self.assertEqual(zip.city, zip.city.title()) self.assertEqual(zip.city, zip.city.title())
city = self.env['res.city'].search(
[('country_id', '=', self.country.id)], limit=1
)
self.assertEqual(city.name, city.name.title())
def test_import_upper(self): def test_import_upper(self):
self.wizard.letter_case = 'upper' self.wizard.letter_case = 'upper'
self.wizard.with_context(max_import=1).run_import() self.wizard.with_context(max_import=1).run_import()
@ -61,3 +87,8 @@ class TestBaseLocationGeonamesImport(common.SavepointCase):
[('country_id', '=', self.country.id)], limit=1 [('country_id', '=', self.country.id)], limit=1
) )
self.assertEqual(zip.city, zip.city.upper()) self.assertEqual(zip.city, zip.city.upper())
city = self.env['res.city'].search(
[('country_id', '=', self.country.id)], limit=1
)
self.assertEqual(city.name, city.name.upper())

127
base_location_geonames_import/wizard/geonames_import.py

@ -1,22 +1,21 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# © 2014-2016 Akretion (Alexis de Lattre <alexis.delattre@akretion.com>)
# © 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
# © 2016 Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
# Copyright 2014-2016 Akretion (Alexis de Lattre
# <alexis.delattre@akretion.com>)
# Copyright 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
# Copyright 2016 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# Copyright 2017 Eficent Business and IT Consulting Services, S.L.
# <contact@eficent.com>
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
from odoo import _, api, fields, models, tools from odoo import _, api, fields, models, tools
from odoo.exceptions import UserError from odoo.exceptions import UserError
import requests import requests
import tempfile import tempfile
import StringIO
import io
import zipfile import zipfile
import os import os
import logging import logging
try:
import unicodecsv
except ImportError:
unicodecsv = None
import csv
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -27,6 +26,12 @@ class BetterZipGeonamesImport(models.TransientModel):
_rec_name = 'country_id' _rec_name = 'country_id'
country_id = fields.Many2one('res.country', 'Country', required=True) country_id = fields.Many2one('res.country', 'Country', required=True)
enforce_cities = fields.Boolean(string='Enforce Cities',
help='The city will be created as a '
'separate entity.',
related='country_id.enforce_cities',
readonly=True)
letter_case = fields.Selection([ letter_case = fields.Selection([
('unchanged', 'Unchanged'), ('unchanged', 'Unchanged'),
('title', 'Title Case'), ('title', 'Title Case'),
@ -46,26 +51,46 @@ class BetterZipGeonamesImport(models.TransientModel):
return city return city
@api.model @api.model
def _domain_search_better_zip(self, row, country):
return [('name', '=', row[1]),
('city', '=', self.transform_city_name(row[2], country)),
def _domain_search_res_city(self, row, country):
return [('name', '=', self.transform_city_name(row[2], country)),
('country_id', '=', country.id)] ('country_id', '=', country.id)]
@api.model @api.model
def _prepare_better_zip(self, row, country):
def _domain_search_better_zip(self, row, country, res_city):
domain = [('name', '=', row[1]),
('city', '=', self.transform_city_name(row[2], country)),
('country_id', '=', country.id)]
if res_city:
domain += [('city_id', '=', res_city.id)]
return domain
@api.model
def _prepare_res_city(self, row, country):
state = self.select_or_create_state(row, country)
vals = {
'name': self.transform_city_name(row[2], country),
'state_id': state.id,
'country_id': country.id,
}
return vals
@api.model
def _prepare_better_zip(self, row, country, res_city):
state = self.select_or_create_state(row, country) state = self.select_or_create_state(row, country)
city_name = self.transform_city_name(row[2], country)
vals = { vals = {
'name': row[1], 'name': row[1],
'city': self.transform_city_name(row[2], country),
'city_id': res_city and res_city.id or False,
'city': res_city and res_city.name or city_name,
'state_id': state.id, 'state_id': state.id,
'country_id': country.id, 'country_id': country.id,
'latitude': row[9], 'latitude': row[9],
'longitude': row[10], 'longitude': row[10],
}
}
return vals return vals
@api.model @api.model
def create_better_zip(self, row, country):
def create_better_zip(self, row, country, res_city):
if row[0] != country.code: if row[0] != country.code:
raise UserError( raise UserError(
_("The country code inside the file (%s) doesn't " _("The country code inside the file (%s) doesn't "
@ -81,16 +106,46 @@ class BetterZipGeonamesImport(models.TransientModel):
if row[1] and row[2]: if row[1] and row[2]:
zip_model = self.env['res.better.zip'] zip_model = self.env['res.better.zip']
zips = zip_model.search(self._domain_search_better_zip( zips = zip_model.search(self._domain_search_better_zip(
row, country))
row, country, res_city))
if zips: if zips:
return zips[0] return zips[0]
else: else:
vals = self._prepare_better_zip(row, country)
vals = self._prepare_better_zip(row, country, res_city)
if vals: if vals:
logger.debug('Creating res.better.zip %s' % vals['name'])
return zip_model.create(vals) return zip_model.create(vals)
else: # pragma: no cover else: # pragma: no cover
return False return False
@api.model
def create_res_city(self, row, country):
if row[0] != country.code:
raise UserError(
_("The country code inside the file (%s) doesn't "
"correspond to the selected country (%s).")
% (row[0], country.code))
logger.debug('Processing city creation for ZIP = %s - City = %s' %
(row[1], row[2]))
if self.letter_case == 'title':
row[2] = row[2].title()
row[3] = row[3].title()
elif self.letter_case == 'upper':
row[2] = row[2].upper()
row[3] = row[3].upper()
if row[2]:
res_city_model = self.env['res.city']
res_cities = res_city_model.search(self._domain_search_res_city(
row, country))
if res_cities:
return res_cities[0]
else:
vals = self._prepare_res_city(row, country)
if vals:
logger.debug('Creating res.city %s' % vals['name'])
return res_city_model.create(vals)
else: # pragma: no cover
return False
@tools.ormcache('country_id', 'code') @tools.ormcache('country_id', 'code')
def _get_state(self, country_id, code, name): def _get_state(self, country_id, code, name):
state = self.env['res.country.state'].search( state = self.env['res.country.state'].search(
@ -117,6 +172,7 @@ class BetterZipGeonamesImport(models.TransientModel):
def run_import(self): def run_import(self):
self.ensure_one() self.ensure_one()
zip_model = self.env['res.better.zip'] zip_model = self.env['res.better.zip']
res_city_model = self.env['res.city']
country_code = self.country_id.code country_code = self.country_id.code
config_url = self.env['ir.config_parameter'].get_param( config_url = self.env['ir.config_parameter'].get_param(
'geonames.url', 'geonames.url',
@ -129,19 +185,32 @@ class BetterZipGeonamesImport(models.TransientModel):
_('Got an error %d when trying to download the file %s.') _('Got an error %d when trying to download the file %s.')
% (res_request.status_code, url)) % (res_request.status_code, url))
# Store current record list # Store current record list
res_cities_to_delete = res_city_model
zips_to_delete = zip_model.search( zips_to_delete = zip_model.search(
[('country_id', '=', self.country_id.id)]) [('country_id', '=', self.country_id.id)])
f_geonames = zipfile.ZipFile(StringIO.StringIO(res_request.content))
tempdir = tempfile.mkdtemp(prefix='openerp')
if self.enforce_cities:
res_cities_to_delete = res_city_model.search(
[('country_id', '=', self.country_id.id)])
f_geonames = zipfile.ZipFile(io.BytesIO(res_request.content))
tempdir = tempfile.mkdtemp(prefix='odoo')
f_geonames.extract('%s.txt' % country_code, tempdir) f_geonames.extract('%s.txt' % country_code, tempdir)
logger.info('The geonames zipfile has been decompressed') logger.info('The geonames zipfile has been decompressed')
data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r')
data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r',
encoding='utf-8')
data_file.seek(0) data_file.seek(0)
logger.info('Starting to create the better zip entries')
logger.info('Starting to create the cities and/or better zip entries')
max_import = self.env.context.get('max_import', 0) max_import = self.env.context.get('max_import', 0)
reader = unicodecsv.reader(data_file, encoding='utf-8', delimiter=' ')
reader = csv.reader(data_file, delimiter=' ')
for i, row in enumerate(reader): for i, row in enumerate(reader):
zip_code = self.create_better_zip(row, self.country_id)
res_city = False
if self.enforce_cities:
res_city = self.create_res_city(row, self.country_id)
if res_city in res_cities_to_delete:
res_cities_to_delete -= res_city
zip_code = self.create_better_zip(row, self.country_id,
res_city)
if zip_code in zips_to_delete: if zip_code in zips_to_delete:
zips_to_delete -= zip_code zips_to_delete -= zip_code
if max_import and (i + 1) == max_import: if max_import and (i + 1) == max_import:
@ -151,7 +220,11 @@ class BetterZipGeonamesImport(models.TransientModel):
zips_to_delete.unlink() zips_to_delete.unlink()
logger.info('%d better zip entries deleted for country %s' % logger.info('%d better zip entries deleted for country %s' %
(len(zips_to_delete), self.country_id.name)) (len(zips_to_delete), self.country_id.name))
if res_cities_to_delete and not max_import:
res_cities_to_delete.unlink()
logger.info('%d res.city entries deleted for country %s' %
(len(res_cities_to_delete), self.country_id.name))
logger.info( logger.info(
'The wizard to create better zip entries from geonames '
'has been successfully completed.')
'The wizard to create cities and/or better zip entries from '
'geonames has been successfully completed.')
return True return True

1
base_location_geonames_import/wizard/geonames_import_view.xml

@ -13,6 +13,7 @@
</group> </group>
<group name="main"> <group name="main">
<field name="country_id"/> <field name="country_id"/>
<field name="enforce_cities"/>
<field name="letter_case"/> <field name="letter_case"/>
</group> </group>
<footer> <footer>

Loading…
Cancel
Save