You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

259 lines
9.6 KiB

  1. # Copyright 2014-2016 Akretion (Alexis de Lattre
  2. # <alexis.delattre@akretion.com>)
  3. # Copyright 2014 Lorenzo Battistini <lorenzo.battistini@agilebg.com>
  4. # Copyright 2017 Eficent Business and IT Consulting Services, S.L.
  5. # <contact@eficent.com>
  6. # Copyright 2018 Aitor Bouzas <aitor.bouzas@adaptivecity.com>
  7. # Copyright 2016-2019 Tecnativa - Pedro M. Baeza
  8. # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
  9. from odoo import _, api, fields, models
  10. from odoo.exceptions import UserError
  11. import requests
  12. import tempfile
  13. import io
  14. import zipfile
  15. import os
  16. import logging
  17. import csv
  18. logger = logging.getLogger(__name__)
  19. class CityZipGeonamesImport(models.TransientModel):
  20. _name = 'city.zip.geonames.import'
  21. _description = 'Import City Zips from Geonames'
  22. _rec_name = 'country_id'
  23. country_id = fields.Many2one('res.country', 'Country', required=True)
  24. code_row_index = fields.Integer(
  25. related='country_id.geonames_state_code_column',
  26. readonly=True)
  27. name_row_index = fields.Integer(
  28. related='country_id.geonames_state_name_column')
  29. letter_case = fields.Selection([
  30. ('unchanged', 'Unchanged'),
  31. ('title', 'Title Case'),
  32. ('upper', 'Upper Case'),
  33. ], string='Letter Case', default='unchanged',
  34. help="Converts retreived city and state names to Title Case "
  35. "(upper case on each first letter of a word) or Upper Case "
  36. "(all letters upper case).")
  37. @api.model
  38. def transform_city_name(self, city, country):
  39. """Override it for transforming city name (if needed)
  40. :param city: Original city name
  41. :param country: Country record
  42. :return: Transformed city name
  43. """
  44. res = city
  45. if self.letter_case == 'title':
  46. res = city.title()
  47. elif self.letter_case == 'upper':
  48. res = city.upper()
  49. return res
  50. @api.model
  51. def _domain_search_res_city(self, row, country):
  52. return [('name', '=', self.transform_city_name(row[2], country)),
  53. ('country_id', '=', country.id)]
  54. @api.model
  55. def _domain_search_city_zip(self, row, res_city):
  56. domain = [('name', '=', row[1])]
  57. if res_city:
  58. domain += [('city_id', '=', res_city.id)]
  59. return domain
  60. @api.model
  61. def select_state(self, row, country):
  62. code = row[self.code_row_index or 4]
  63. return self.env['res.country.state'].search(
  64. [('country_id', '=', country.id),
  65. ('code', '=', code)], limit=1,
  66. )
  67. @api.model
  68. def select_city(self, row, country):
  69. res_city_model = self.env['res.city']
  70. return res_city_model.search(self._domain_search_res_city(
  71. row, country), limit=1)
  72. @api.model
  73. def select_zip(self, row, country):
  74. city = self.select_city(row, country)
  75. return self.env['res.city.zip'].search(self._domain_search_city_zip(
  76. row, city))
  77. @api.model
  78. def prepare_state(self, row, country):
  79. return {
  80. 'name': row[self.name_row_index or 3],
  81. 'code': row[self.code_row_index or 4],
  82. 'country_id': country.id,
  83. }
  84. @api.model
  85. def prepare_city(self, row, country, state_id):
  86. vals = {
  87. 'name': self.transform_city_name(row[2], country),
  88. 'state_id': state_id,
  89. 'country_id': country.id,
  90. }
  91. return vals
  92. @api.model
  93. def prepare_zip(self, row, city_id):
  94. vals = {
  95. 'name': row[1],
  96. 'city_id': city_id,
  97. }
  98. return vals
  99. @api.model
  100. def get_and_parse_csv(self):
  101. country_code = self.country_id.code
  102. config_url = self.env['ir.config_parameter'].get_param(
  103. 'geonames.url',
  104. default='http://download.geonames.org/export/zip/%s.zip')
  105. url = config_url % country_code
  106. logger.info('Starting to download %s' % url)
  107. res_request = requests.get(url)
  108. if res_request.status_code != requests.codes.ok:
  109. raise UserError(
  110. _('Got an error %d when trying to download the file %s.')
  111. % (res_request.status_code, url))
  112. f_geonames = zipfile.ZipFile(io.BytesIO(res_request.content))
  113. tempdir = tempfile.mkdtemp(prefix='odoo')
  114. f_geonames.extract('%s.txt' % country_code, tempdir)
  115. data_file = open(os.path.join(tempdir, '%s.txt' % country_code), 'r',
  116. encoding='utf-8')
  117. data_file.seek(0)
  118. reader = csv.reader(data_file, delimiter=' ')
  119. parsed_csv = [row for i, row in enumerate(reader)]
  120. data_file.close()
  121. logger.info('The geonames zipfile has been decompressed')
  122. return parsed_csv
  123. def _create_states(self, parsed_csv, search_states, max_import):
  124. # States
  125. state_vals_list = []
  126. state_dict = {}
  127. for i, row in enumerate(parsed_csv):
  128. if max_import and i == max_import:
  129. break
  130. state = self.select_state(
  131. row, self.country_id) if search_states else False
  132. if not state:
  133. state_vals = self.prepare_state(row, self.country_id)
  134. if state_vals not in state_vals_list:
  135. state_vals_list.append(state_vals)
  136. else:
  137. state_dict[state.code] = state.id
  138. created_states = self.env['res.country.state'].create(state_vals_list)
  139. for i, vals in enumerate(state_vals_list):
  140. state_dict[vals['code']] = created_states[i].id
  141. return state_dict
  142. def _create_cities(self, parsed_csv,
  143. search_cities, max_import, state_dict):
  144. # Cities
  145. city_vals_list = []
  146. city_dict = {}
  147. for i, row in enumerate(parsed_csv):
  148. if max_import and i == max_import:
  149. break
  150. state_id = state_dict[row[self.code_row_index or 4]]
  151. city = self.select_city(
  152. row, self.country_id) if search_cities else False
  153. if not city:
  154. city_vals = self.prepare_city(
  155. row, self.country_id, state_id)
  156. if city_vals not in city_vals_list:
  157. city_vals_list.append(city_vals)
  158. else:
  159. city_dict[(city.name, state_id)] = city.id
  160. created_cities = self.env['res.city'].create(city_vals_list)
  161. for i, vals in enumerate(city_vals_list):
  162. city_dict[(vals['name'], vals['state_id'])] = created_cities[i].id
  163. return city_dict
  164. @api.multi
  165. def run_import(self):
  166. self.ensure_one()
  167. parsed_csv = self.get_and_parse_csv()
  168. return self._process_csv(parsed_csv)
  169. def _process_csv(self, parsed_csv):
  170. state_model = self.env['res.country.state']
  171. zip_model = self.env['res.city.zip']
  172. res_city_model = self.env['res.city']
  173. # Store current record list
  174. current_zips = zip_model.search(
  175. [('city_id.country_id', '=', self.country_id.id)])
  176. search_zips = True and len(current_zips) > 0 or False
  177. current_cities = res_city_model.search(
  178. [('country_id', '=', self.country_id.id)])
  179. search_cities = True and len(current_cities) > 0 or False
  180. current_states = state_model.search(
  181. [('country_id', '=', self.country_id.id)])
  182. search_states = True and len(current_states) > 0 or False
  183. max_import = self.env.context.get('max_import', 0)
  184. logger.info('Starting to create the cities and/or city zip entries')
  185. state_dict = self._create_states(parsed_csv,
  186. search_states, max_import)
  187. city_dict = self._create_cities(parsed_csv,
  188. search_cities, max_import, state_dict)
  189. # Zips
  190. zip_vals_list = []
  191. for i, row in enumerate(parsed_csv):
  192. if max_import and i == max_import:
  193. break
  194. # Don't search if there aren't any records
  195. zip_code = False
  196. if search_zips:
  197. zip_code = self.select_zip(row, self.country_id)
  198. if not zip_code:
  199. state_id = state_dict[row[self.code_row_index or 4]]
  200. city_id = city_dict[(
  201. self.transform_city_name(row[2], self.country_id),
  202. state_id,
  203. )]
  204. zip_vals = self.prepare_zip(row, city_id)
  205. if zip_vals not in zip_vals_list:
  206. zip_vals_list.append(zip_vals)
  207. delete_zips = self.env['res.city.zip'].create(zip_vals_list)
  208. current_zips -= delete_zips
  209. if not max_import:
  210. current_zips.unlink()
  211. logger.info('%d city zip entries deleted for country %s' %
  212. (len(current_zips), self.country_id.name))
  213. # Since we wrapped the entire cities
  214. # creation in a function we need
  215. # to perform a search with city_dict in
  216. # order to know which are the new ones so
  217. # we can delete the old ones
  218. created_cities = res_city_model.search(
  219. [('country_id', '=', self.country_id.id),
  220. ('id', 'in', list(city_dict.values()))]
  221. )
  222. current_cities -= created_cities
  223. current_cities.unlink()
  224. logger.info('%d res.city entries deleted for country %s' %
  225. (len(current_cities), self.country_id.name))
  226. logger.info(
  227. 'The wizard to create cities and/or city zip entries from '
  228. 'geonames has been successfully completed.')
  229. return True