diff --git a/.travis.yml b/.travis.yml index 6341713d1..da5809c6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ virtualenv: system_site_packages: true install: + - pip install requests + - pip install unicodecsv - git clone https://github.com/OCA/maintainer-quality-tools.git ${HOME}/maintainer-quality-tools - export PATH=${HOME}/maintainer-quality-tools/travis:${PATH} - travis_install_nightly ${VERSION} diff --git a/base_location_geonames_import/__init__.py b/base_location_geonames_import/__init__.py new file mode 100644 index 000000000..b63234221 --- /dev/null +++ b/base_location_geonames_import/__init__.py @@ -0,0 +1,23 @@ +# -*- encoding: utf-8 -*- +############################################################################## +# +# Base Location Geonames Import module for OpenERP +# Copyright (C) 2014 Akretion (http://www.akretion.com) +# @author Alexis de Lattre +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +from . import wizard diff --git a/base_location_geonames_import/__openerp__.py b/base_location_geonames_import/__openerp__.py new file mode 100644 index 000000000..f4baecd64 --- /dev/null +++ b/base_location_geonames_import/__openerp__.py @@ -0,0 +1,68 @@ +# -*- encoding: utf-8 -*- +############################################################################## +# +# Base Location Geonames Import module for OpenERP +# Copyright (C) 2014 Akretion (http://www.akretion.com) +# @author Alexis de Lattre +# Copyright (C) 2014 Agile Business Group (http://www.agilebg.com) +# @author Lorenzo Battistini +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + + +{ + 'name': 'Base Location Geonames Import', + 'version': '0.2', + 'category': 'Extra Tools', + 'license': 'AGPL-3', + 'summary': 'Import better zip entries from Geonames', + 'description': """ +Base Location Geonames Import +============================= + +This module adds a wizard to import better zip entries from Geonames +(http://download.geonames.org/export/zip/). +If want want/need to modify the URL, you can set the 'geonames.url' +system parameter. + +When you start the wizard, +it will ask you to select a country. Then, for the selected country, +it will delete all the current better zip entries, download the latest version +of the list of cities from geonames.org and create new better zip entries. + +Please contact Alexis de Lattre from Akretion +for any help or question about this module. + + +Contributors +------------ + +- Alexis de Lattre +- Lorenzo Battistini +""", + 'author': 'Akretion', + 'website': 'http://www.akretion.com', + 'depends': ['base_location'], + 'external_dependencies': {'python': ['requests', 'unicodecsv']}, + 'data': [ + 'wizard/geonames_import_view.xml', + ], + 'test': [ + 'test/import.yml' + ], + 'installable': True, + 'active': False, +} diff --git a/base_location_geonames_import/i18n/base_location_geonames_import.pot b/base_location_geonames_import/i18n/base_location_geonames_import.pot new file mode 100644 index 000000000..7c5782ea3 --- /dev/null +++ b/base_location_geonames_import/i18n/base_location_geonames_import.pot @@ -0,0 +1,68 @@ +# Translation of OpenERP Server. +# This file contains the translation of the following modules: +# * base_location_geonames_import +# +msgid "" +msgstr "" +"Project-Id-Version: OpenERP Server 7.0\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2014-04-11 20:55+0000\n" +"PO-Revision-Date: 2014-04-11 20:55+0000\n" +"Last-Translator: <>\n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: base_location_geonames_import +#: view:better.zip.geonames.import:0 +msgid "Cancel" +msgstr "" + +#. module: base_location_geonames_import +#: field:better.zip.geonames.import,country_id:0 +msgid "Country" +msgstr "" + +#. module: base_location_geonames_import +#: code:addons/base_location_geonames_import/wizard/geonames_import.py:66 +#: code:addons/base_location_geonames_import/wizard/geonames_import.py:90 +#, python-format +msgid "Error:" +msgstr "" + +#. module: base_location_geonames_import +#: view:better.zip.geonames.import:0 +msgid "For the country selected above, this wizard will DELETE ALL THE CURRENT BETTER ZIP ENTRIES, download the latest version of the list of cities from geonames.org and create new better zip entries." +msgstr "" + +#. module: base_location_geonames_import +#: code:addons/base_location_geonames_import/wizard/geonames_import.py:91 +#, python-format +msgid "Got an error %d when trying to download the file %s." +msgstr "" + +#. module: base_location_geonames_import +#: view:better.zip.geonames.import:0 +msgid "Import" +msgstr "" + +#. module: base_location_geonames_import +#: model:ir.model,name:base_location_geonames_import.model_better_zip_geonames_import +msgid "Import Better Zip from Geonames" +msgstr "" + +#. module: base_location_geonames_import +#: view:better.zip.geonames.import:0 +#: model:ir.actions.act_window,name:base_location_geonames_import.better_zip_geonames_import_action +#: model:ir.ui.menu,name:base_location_geonames_import.better_zip_geonames_import_menu +msgid "Import Geonames" +msgstr "" + +#. module: base_location_geonames_import +#: code:addons/base_location_geonames_import/wizard/geonames_import.py:67 +#, python-format +msgid "The country code inside the file (%s) doesn't correspond to the selected country (%s)." +msgstr "" + diff --git a/base_location_geonames_import/i18n/fr.po b/base_location_geonames_import/i18n/fr.po new file mode 100644 index 000000000..699ad7519 --- /dev/null +++ b/base_location_geonames_import/i18n/fr.po @@ -0,0 +1,68 @@ +# Translation of OpenERP Server. +# This file contains the translation of the following modules: +# * base_location_geonames_import +# +msgid "" +msgstr "" +"Project-Id-Version: OpenERP Server 7.0\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2014-04-11 20:56+0000\n" +"PO-Revision-Date: 2014-04-11 20:56+0000\n" +"Last-Translator: <>\n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: base_location_geonames_import +#: view:better.zip.geonames.import:0 +msgid "Cancel" +msgstr "Annuler" + +#. module: base_location_geonames_import +#: field:better.zip.geonames.import,country_id:0 +msgid "Country" +msgstr "Pays" + +#. module: base_location_geonames_import +#: code:addons/base_location_geonames_import/wizard/geonames_import.py:66 +#: code:addons/base_location_geonames_import/wizard/geonames_import.py:90 +#, python-format +msgid "Error:" +msgstr "Erreur :" + +#. module: base_location_geonames_import +#: view:better.zip.geonames.import:0 +msgid "For the country selected above, this wizard will DELETE ALL THE CURRENT BETTER ZIP ENTRIES, download the latest version of the list of cities from geonames.org and create new better zip entries." +msgstr "Pour le pays sélectionné ci-dessus, cet assistant va SUPPRIMER TOUTES LES ENTREES BETTER ZIP, télécharger la dernière version de la liste des villes depuis geonames.org et créer de nouveaux enregistrements better zip." + +#. module: base_location_geonames_import +#: code:addons/base_location_geonames_import/wizard/geonames_import.py:91 +#, python-format +msgid "Got an error %d when trying to download the file %s." +msgstr "Erreur %d reçue suite à la tentative de téléchargement du fichier %s." + +#. module: base_location_geonames_import +#: view:better.zip.geonames.import:0 +msgid "Import" +msgstr "Importer" + +#. module: base_location_geonames_import +#: model:ir.model,name:base_location_geonames_import.model_better_zip_geonames_import +msgid "Import Better Zip from Geonames" +msgstr "Import Better Zip from Geonames" + +#. module: base_location_geonames_import +#: view:better.zip.geonames.import:0 +#: model:ir.actions.act_window,name:base_location_geonames_import.better_zip_geonames_import_action +#: model:ir.ui.menu,name:base_location_geonames_import.better_zip_geonames_import_menu +msgid "Import Geonames" +msgstr "Importer Geonames" + +#. module: base_location_geonames_import +#: code:addons/base_location_geonames_import/wizard/geonames_import.py:67 +#, python-format +msgid "The country code inside the file (%s) doesn't correspond to the selected country (%s)." +msgstr "Le code pays utilisé à l'intérieur du fichier (%s) ne correspond pas au pays sélectionné (%s)." + diff --git a/base_location_geonames_import/test/import.yml b/base_location_geonames_import/test/import.yml new file mode 100644 index 000000000..66243d24a --- /dev/null +++ b/base_location_geonames_import/test/import.yml @@ -0,0 +1,54 @@ +- + I create the wizard +- + !record {model: better.zip.geonames.import, id: import_wizard_1, view: better_zip_geonames_import_form}: + country_id: base.it +- + I run the import +- + !python {model: better.zip.geonames.import}: | + self.run_import(cr, uid, [ref('import_wizard_1')], context=context) +- + I check the data +- + !python {model: res.better.zip}: | + state_obj = self.pool['res.country.state'] + state_ids = state_obj.search(cr, uid, [ + ('code', '=', 'LG'), + ('country_id', '=', ref('base.it')), + ], context=context) + assert len(state_ids) == 1, "There must be 1 LG" + zip_ids = self.search(cr, uid, [ + ('name', '=', '16017'), + ('city', '=', 'Isola Del Cantone'), + ('state_id', '=', state_ids[0]), + ('country_id', '=', ref('base.it')) + ], context=context) + assert len(zip_ids) == 1, "There must be 1 'Isola Del Cantone'" +- + I create the wizard again +- + !record {model: better.zip.geonames.import, id: import_wizard_2, view: better_zip_geonames_import_form}: + country_id: base.it +- + I run the import again +- + !python {model: better.zip.geonames.import}: | + self.run_import(cr, uid, [ref('import_wizard_2')], context=context) +- + I check the data +- + !python {model: res.better.zip}: | + state_obj = self.pool['res.country.state'] + state_ids = state_obj.search(cr, uid, [ + ('code', '=', 'LG'), + ('country_id', '=', ref('base.it')), + ], context=context) + assert len(state_ids) == 1, "There must be 1 LG" + zip_ids = self.search(cr, uid, [ + ('name', '=', '16017'), + ('city', '=', 'Isola Del Cantone'), + ('state_id', '=', state_ids[0]), + ('country_id', '=', ref('base.it')) + ], context=context) + assert len(zip_ids) == 1, "There must be 1 'Isola Del Cantone'" diff --git a/base_location_geonames_import/wizard/__init__.py b/base_location_geonames_import/wizard/__init__.py new file mode 100644 index 000000000..2c9462e69 --- /dev/null +++ b/base_location_geonames_import/wizard/__init__.py @@ -0,0 +1,23 @@ +# -*- encoding: utf-8 -*- +############################################################################## +# +# Base Location Geonames Import module for OpenERP +# Copyright (C) 2014 Akretion (http://www.akretion.com) +# @author Alexis de Lattre +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +from . import geonames_import diff --git a/base_location_geonames_import/wizard/geonames_import.py b/base_location_geonames_import/wizard/geonames_import.py new file mode 100644 index 000000000..9fb2cf355 --- /dev/null +++ b/base_location_geonames_import/wizard/geonames_import.py @@ -0,0 +1,131 @@ +# -*- encoding: utf-8 -*- +############################################################################## +# +# Base Location Geonames Import module for OpenERP +# Copyright (C) 2014 Akretion (http://www.akretion.com) +# @author Alexis de Lattre +# Copyright (C) 2014 Agile Business Group (http://www.agilebg.com) +# @author Lorenzo Battistini +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +from openerp import models, fields, api, _ +from openerp.exceptions import Warning +import requests +import tempfile +import StringIO +import unicodecsv +import zipfile +import os +import logging + +logger = logging.getLogger(__name__) + + +class better_zip_geonames_import(models.TransientModel): + _name = 'better.zip.geonames.import' + _description = 'Import Better Zip from Geonames' + _rec_name = 'country_id' + + country_id = fields.Many2one('res.country', 'Country', required=True) + + @api.model + def _prepare_better_zip(self, row, country_id): + state = self.select_or_create_state(row, country_id) + vals = { + 'name': row[1], + 'city': row[2], + 'state_id': state.id, + 'country_id': country_id, + } + return vals + + @api.model + def create_better_zip( + self, row, country_id, country_code): + bzip_id = False + if row[0] != country_code: + raise Warning( + _('Error:'), + _("The country code inside the file (%s) doesn't " + "correspond to the selected country (%s).") + % (row[0], country_code)) + logger.debug('ZIP = %s - City = %s' % (row[1], row[2])) + if row[1] and row[2]: + vals = self._prepare_better_zip(row, country_id) + if vals: + bzip_id = self.env['res.better.zip'].create(vals) + return bzip_id + + @api.model + def select_or_create_state( + self, row, country_id, code_row_index=4, name_row_index=3 + ): + states = self.env['res.country.state'].search([ + ('country_id', '=', country_id), + ('code', '=', row[code_row_index]), + ]) + if len(states) > 1: + raise Warning( + _("Too many states with code %s for counrty %s") + % (row[code_row_index], country_id)) + if len(states) == 1: + return states[0] + else: + return self.env['res.country.state'].create({ + 'name': row[name_row_index], + 'code': row[code_row_index], + 'country_id': country_id + }) + + @api.one + def run_import(self): + bzip_obj = self.env['res.better.zip'] + country_id = self.country_id.id + country_code = self.country_id.code.upper() + config_url = self.pool['ir.config_parameter'].get_param( + self._cr, self._uid, 'geonames.url', + default='http://download.geonames.org/export/zip/%s.zip') + url = config_url % country_code + logger.info('Starting to download %s' % url) + res_request = requests.get(url) + if res_request.status_code != requests.codes.ok: + raise Warning( + _('Error:'), + _('Got an error %d when trying to download the file %s.') + % (res_request.status_code, url)) + bzip_ids_to_delete = bzip_obj.search([('country_id', '=', country_id)]) + if bzip_ids_to_delete: + bzip_ids_to_delete.unlink() + logger.info( + '%d better zip entries deleted for country %s' + % (len(bzip_ids_to_delete), self.country_id.name)) + f_geonames = zipfile.ZipFile(StringIO.StringIO(res_request.content)) + tempdir = tempfile.mkdtemp(prefix='openerp') + f_geonames.extract('%s.txt' % country_code, tempdir) + logger.info('The geonames zipfile has been decompressed') + data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r') + data_file.seek(0) + logger.info( + 'Starting to create the better zip entries') + for row in unicodecsv.reader( + data_file, encoding='utf-8', delimiter=' '): + self.create_better_zip(row, country_id, country_code) + data_file.close() + logger.info( + 'The wizard to create better zip entries from geonames ' + 'has been successfully completed.') + return True diff --git a/base_location_geonames_import/wizard/geonames_import_view.xml b/base_location_geonames_import/wizard/geonames_import_view.xml new file mode 100644 index 000000000..efd37057d --- /dev/null +++ b/base_location_geonames_import/wizard/geonames_import_view.xml @@ -0,0 +1,44 @@ + + + + + + + + asterisk.server.company + better.zip.geonames.import + +
+ + + +
+
+
+ + + + + Import Geonames + better.zip.geonames.import + form + form + new + + + + + +