OCA
/
server-tools
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

[10.0][MIG][base_import_match] Migration and update 

Includes:

- Normal migration steps.
- Usage of brand new `_inherit = "base"` in Odoo 10, which implies
  removing a lot of monkey-patching code.
- Log a warning when multiple matches are found.
pull/871/head
Jairo Llopis 7 years ago
committed by Pedro M. Baeza
parent
c3cb1562b8
commit
ab0ceebd86
6 changed files with 96 additions and 165 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      base_import_match/README.rst
  2. 2
      base_import_match/__manifest__.py
  3. 1
      base_import_match/models/__init__.py
  4. 54
      base_import_match/models/base.py
  5. 196
      base_import_match/models/base_import.py
  6. 2
      base_import_match/tests/test_import.py

4
base_import_match/README.rst
View File

@ -61,7 +61,7 @@ To configure this module, you need to:
#. Go to *Settings > Technical > Database Structure > Import Match*. #. Go to *Settings > Technical > Database Structure > Import Match*.
#. *Create*. #. *Create*.
#. Choose a *Model*. #. Choose a *Model*.
#. Choose the *Fields* that conform an unique key in that model.
#. Choose the *Fields* that conform a unique key in that model.
#. If the rule must be used only for certain imported values, check #. If the rule must be used only for certain imported values, check
*Conditional* and enter the **exact string** that is going to be imported *Conditional* and enter the **exact string** that is going to be imported
in *Imported value*. in *Imported value*.
@ -84,7 +84,7 @@ To use this module, you need to:
.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas .. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas
:alt: Try me on Runbot :alt: Try me on Runbot
:target: https://runbot.odoo-community.org/runbot/149/9.0
:target: https://runbot.odoo-community.org/runbot/149/10.0
Known Issues / Roadmap Known Issues / Roadmap
====================== ======================

2
base_import_match/__openerp__.py → base_import_match/__manifest__.py
View File

@ -5,7 +5,7 @@
{ {
"name": "Base Import Match", "name": "Base Import Match",
"summary": "Try to avoid duplicates before importing", "summary": "Try to avoid duplicates before importing",
"version": "9.0.1.0.0",
"version": "10.0.1.0.0",
"category": "Tools", "category": "Tools",
"website": "https://tecnativa.com", "website": "https://tecnativa.com",
"author": "Grupo ESOC Ingeniería de Servicios," "author": "Grupo ESOC Ingeniería de Servicios,"

1
base_import_match/models/__init__.py
View File

@ -3,4 +3,5 @@
# Copyright 2016 Tecnativa - Vicent Cubells # Copyright 2016 Tecnativa - Vicent Cubells
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
from . import base
from . import base_import from . import base_import

54
base_import_match/models/base.py
View File

@ -0,0 +1,54 @@
# -*- coding: utf-8 -*-
# Copyright 2017 Jairo Llopis <jairo.llopis@tecnativa.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
from odoo import api, models
class Base(models.AbstractModel):
_inherit = "base"
@api.model
def load(self, fields, data):
"""Try to identify rows by other pseudo-unique keys.
It searches for rows that have no XMLID specified, and gives them
one if any :attr:`~.field_ids` combination is found. With a valid
XMLID in place, Odoo will understand that it must *update* the
record instead of *creating* a new one.
"""
# We only need to patch this call if there are usable rules for it
if self.env["base_import.match"]._usable_rules(self._name, fields):
newdata = list()
# Data conversion to ORM format
import_fields = map(models.fix_import_export_id_paths, fields)
converted_data = self._convert_records(
self._extract_records(import_fields, data))
# Mock Odoo to believe the user is importing the ID field
if "id" not in fields:
fields.append("id")
import_fields.append(["id"])
# Needed to match with converted data field names
clean_fields = [f[0] for f in import_fields]
for dbid, xmlid, record, info in converted_data:
row = dict(zip(clean_fields, data[info["record"]]))
match = self
if xmlid:
# Skip rows with ID, they do not need all this
row["id"] = xmlid
continue
elif dbid:
# Find the xmlid for this dbid
match = self.browse(dbid)
else:
# Store records that match a combination
match = self.env["base_import.match"]._match_find(
self, record, row)
# Give a valid XMLID to this row if a match was found
row["id"] = (match._BaseModel__export_xml_id()
if match else row.get("id", u""))
# Store the modified row, in the same order as fields
newdata.append(tuple(row[f] for f in clean_fields))
# We will import the patched data to get updates on matches
data = newdata
# Normal method handles the rest of the job
return super(Base, self).load(fields, data)

196
base_import_match/models/base_import.py
View File

@ -2,8 +2,10 @@
# Copyright 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis # Copyright 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis
# Copyright 2016 Tecnativa - Vicent Cubells # Copyright 2016 Tecnativa - Vicent Cubells
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
from openerp import api, fields, models
from openerp import SUPERUSER_ID # TODO remove in v10
import logging
from odoo import api, fields, models, tools
_logger = logging.getLogger(__name__)
class BaseImportMatch(models.Model): class BaseImportMatch(models.Model):
@ -34,63 +36,18 @@ class BaseImportMatch(models.Model):
required=True, required=True,
help="Fields that will define an unique key.") help="Fields that will define an unique key.")
@api.multi
@api.onchange("model_id") @api.onchange("model_id")
def _onchange_model_id(self): def _onchange_model_id(self):
self.field_ids.unlink()
@api.model
def create(self, vals):
"""Wrap the model after creation."""
result = super(BaseImportMatch, self).create(vals)
self._load_autopatch(result.model_name)
return result
@api.multi
def unlink(self):
"""Unwrap the model after deletion."""
models = set(self.mapped("model_name"))
result = super(BaseImportMatch, self).unlink()
for model in models:
self._load_autopatch(model)
return result
@api.multi
def write(self, vals):
"""Wrap the model after writing."""
result = super(BaseImportMatch, self).write(vals)
self.field_ids = False
if "model_id" in vals or "model_name" in vals:
for s in self:
self._load_autopatch(s.model_name)
return result
# TODO convert to @api.model_cr in v10
def _register_hook(self, cr):
"""Autopatch on init."""
models = set(
self.browse(
cr,
SUPERUSER_ID,
self.search(cr, SUPERUSER_ID, list()))
.mapped("model_name"))
for model in models:
self._load_autopatch(cr, SUPERUSER_ID, model)
@api.multi
@api.depends("model_id", "field_ids") @api.depends("model_id", "field_ids")
def _compute_name(self): def _compute_name(self):
"""Automatic self-descriptive name for the setting records.""" """Automatic self-descriptive name for the setting records."""
for s in self:
s.name = u"{}: {}".format(
s.model_id.display_name,
" + ".join(
s.field_ids.mapped(
lambda r: (
(u"{} ({})" if r.conditional else u"{}").format(
r.field_id.name,
r.imported_value)))))
for one in self:
one.name = u"{}: {}".format(
one.model_id.display_name,
" + ".join(one.field_ids.mapped("display_name")),
)
@api.model @api.model
def _match_find(self, model, converted_row, imported_row): def _match_find(self, model, converted_row, imported_row):
@ -100,12 +57,12 @@ class BaseImportMatch(models.Model):
imported data, and return a match for the first rule that returns a imported data, and return a match for the first rule that returns a
single result. single result.
:param openerp.models.Model model:
:param odoo.models.Model model:
Model object that is being imported. Model object that is being imported.
:param dict converted_row: :param dict converted_row:
Row converted to Odoo api format, like the 3rd value that Row converted to Odoo api format, like the 3rd value that
:meth:`openerp.models.Model._convert_records` returns.
:meth:`odoo.models.Model._convert_records` returns.
:param dict imported_row: :param dict imported_row:
Row as it is being imported, in format:: Row as it is being imported, in format::
@ -116,18 +73,16 @@ class BaseImportMatch(models.Model):
... ...
} }
:return openerp.models.Model:
:return odoo.models.Model:
Return a dataset with one single match if it was found, or an Return a dataset with one single match if it was found, or an
empty dataset if none or multiple matches were found. empty dataset if none or multiple matches were found.
""" """
# Get usable rules to perform matches # Get usable rules to perform matches
usable = self._usable_for_load(model._name, converted_row.keys())
usable = self._usable_rules(model._name, converted_row)
# Traverse usable combinations # Traverse usable combinations
for combination in usable: for combination in usable:
combination_valid = True combination_valid = True
domain = list() domain = list()
for field in combination.field_ids: for field in combination.field_ids:
# Check imported value if it is a conditional field # Check imported value if it is a conditional field
if field.conditional: if field.conditional:
@ -135,114 +90,26 @@ class BaseImportMatch(models.Model):
if imported_row[field.name] != field.imported_value: if imported_row[field.name] != field.imported_value:
combination_valid = False combination_valid = False
break break
domain.append((field.name, "=", converted_row[field.name])) domain.append((field.name, "=", converted_row[field.name]))
if not combination_valid: if not combination_valid:
continue continue
match = model.search(domain) match = model.search(domain)
# When a single match is found, stop searching # When a single match is found, stop searching
if len(match) == 1: if len(match) == 1:
return match return match
elif match:
_logger.warning(
"Found multiple matches for model %s and domain %s; "
"falling back to default behavior (create new record)",
model._name,
domain,
)
# Return an empty match if none or multiple was found # Return an empty match if none or multiple was found
return model return model
@api.model @api.model
def _load_wrapper(self):
"""Create a new load patch method."""
@api.model
def wrapper(self, fields, data):
"""Try to identify rows by other pseudo-unique keys.
It searches for rows that have no XMLID specified, and gives them
one if any :attr:`~.field_ids` combination is found. With a valid
XMLID in place, Odoo will understand that it must *update* the
record instead of *creating* a new one.
"""
newdata = list()
# Data conversion to ORM format
import_fields = map(models.fix_import_export_id_paths, fields)
converted_data = self._convert_records(
self._extract_records(import_fields, data))
# Mock Odoo to believe the user is importing the ID field
if "id" not in fields:
fields.append("id")
import_fields.append(["id"])
# Needed to match with converted data field names
clean_fields = [f[0] for f in import_fields]
for dbid, xmlid, record, info in converted_data:
row = dict(zip(clean_fields, data[info["record"]]))
match = self
if xmlid:
# Skip rows with ID, they do not need all this
row["id"] = xmlid
elif dbid:
# Find the xmlid for this dbid
match = self.browse(dbid)
else:
# Store records that match a combination
match = self.env["base_import.match"]._match_find(
self, record, row)
# Give a valid XMLID to this row if a match was found
row["id"] = (match._BaseModel__export_xml_id()
if match else row.get("id", u""))
# Store the modified row, in the same order as fields
newdata.append(tuple(row[f] for f in clean_fields))
# Leave the rest to Odoo itself
del data
return wrapper.origin(self, fields, newdata)
# Flag to avoid confusions with other possible wrappers
wrapper.__base_import_match = True
return wrapper
@api.model
def _load_autopatch(self, model_name):
"""[Un]apply patch automatically."""
self._load_unpatch(model_name)
if self.search([("model_name", "=", model_name)]):
self._load_patch(model_name)
@api.model
def _load_patch(self, model_name):
"""Apply patch for :param:`model_name`'s load method.
:param str model_name:
Model technical name, such as ``res.partner``.
"""
self.env[model_name]._patch_method(
"load", self._load_wrapper())
@api.model
def _load_unpatch(self, model_name):
"""Apply patch for :param:`model_name`'s load method.
:param str model_name:
Model technical name, such as ``res.partner``.
"""
model = self.env[model_name]
# Unapply patch only if there is one
try:
if model.load.__base_import_match:
model._revert_method("load")
except AttributeError:
pass
@api.model
def _usable_for_load(self, model_name, fields):
@tools.ormcache("model_name", "fields")
def _usable_rules(self, model_name, fields):
"""Return a set of elements usable for calling ``load()``. """Return a set of elements usable for calling ``load()``.
:param str model_name: :param str model_name:
@ -251,15 +118,16 @@ class BaseImportMatch(models.Model):
:param list(str|bool) fields: :param list(str|bool) fields:
List of field names being imported. List of field names being imported.
:return bool:
Indicates if we should patch its load method.
""" """
result = self result = self
available = self.search([("model_name", "=", model_name)]) available = self.search([("model_name", "=", model_name)])
# Use only criteria with all required fields to match # Use only criteria with all required fields to match
for record in available: for record in available:
if all(f.name in fields for f in record.field_ids): if all(f.name in fields for f in record.field_ids):
result += record
result |= record
return result return result
@ -291,7 +159,15 @@ class BaseImportMatchField(models.Model):
"string, and comparison is case-sensitive so if you set 'True', " "string, and comparison is case-sensitive so if you set 'True', "
"it will NOT match '1' nor 'true', only EXACTLY 'True'.") "it will NOT match '1' nor 'true', only EXACTLY 'True'.")
@api.multi
@api.depends("conditional", "field_id", "imported_value")
def _compute_display_name(self):
for one in self:
pattern = u"{name} ({cond})" if one.conditional else u"{name}"
one.display_name = pattern.format(
name=one.field_id.name,
cond=one.imported_value,
)
@api.onchange("field_id", "match_id", "conditional", "imported_value") @api.onchange("field_id", "match_id", "conditional", "imported_value")
def _onchange_match_id_name(self): def _onchange_match_id_name(self):
"""Update match name.""" """Update match name."""

2
base_import_match/tests/test_import.py
View File

@ -4,7 +4,7 @@
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
from os import path from os import path
from openerp.tests.common import TransactionCase
from odoo.tests.common import TransactionCase
PATH = path.join(path.dirname(__file__), "import_data", "%s.csv") PATH = path.join(path.dirname(__file__), "import_data", "%s.csv")

Write Preview
Loading…
Cancel
Save