diff --git a/base_import_match/README.rst b/base_import_match/README.rst index 6066502a6..f1c657728 100644 --- a/base_import_match/README.rst +++ b/base_import_match/README.rst @@ -9,8 +9,8 @@ Base Import Match By default, when importing data (like CSV import) with the ``base_import`` module, Odoo follows this rule: -#. If you import the XMLID of a record, make an **update**. -#. If you do not, **create** a new record. +- If you import the XMLID of a record, make an **update**. +- If you do not, **create** a new record. This module allows you to set additional rules to match if a given import is an update or a new record. @@ -21,21 +21,31 @@ name, VAT, email, etc. After installing this module, the import logic will be changed to: -#. If you import the XMLID of a record, make an **update**. -#. If you do not: - #. If there are import match rules for the model you are importing: - #. Discard the rules that require fields you are not importing. - #. Traverse the remaining rules one by one in order to find a match in - the database. - #. If one match is found: - #. Stop traversing the rest of valid rules. - #. **Update** that record. - #. If zero or multiple matches are found: - #. Continue with the next rule. - #. If all rules are exhausted and no single match is found: - #. **Create** a new record. - #. If there are no match rules for your model: - #. **Create** a new record. +- If you import the XMLID of a record, make an **update**. +- If you do not: + - If there are import match rules for the model you are importing: + - Discard the rules that require fields you are not importing. + - Traverse the remaining rules one by one in order to find a match in + the database. + - Skip the rule if it requires a special condition that is not + satisfied. + - If one match is found: + - Stop traversing the rest of valid rules. + - **Update** that record. + - If zero or multiple matches are found: + - Continue with the next rule. + - If all rules are exhausted and no single match is found: + - **Create** a new record. + - If there are no match rules for your model: + - **Create** a new record. + +By default 2 rules are installed for production instances: + +- One rule that will allow you to update companies based on their VAT, when + ``is_company`` is ``True``. +- One rule that will allow you to update users based on their login. + +In demo instances there are more examples. Configuration ============= @@ -46,6 +56,12 @@ To configure this module, you need to: #. *Create*. #. Choose a *Model*. #. Choose the *Fields* that conform an unique key in that model. +#. If the rule must be used only for certain imported values, check + *Conditional* and enter the **exact string** that is going to be imported + in *Imported value*. + #. Keep in mind that the match here is evaluated as a case sensitive + **text string** always. If you enter e.g. ``True``, it will match that + string, but will not match ``1`` or ``true``. #. *Save*. In that list view, you can sort rules by drag and drop. @@ -63,15 +79,11 @@ To use this module, you need to: :alt: Try me on Runbot :target: https://runbot.odoo-community.org/runbot/149/8.0 -Roadmap / Known Issues +Known Issues / Roadmap ====================== -* Add a filter to let you apply some rules only to incoming imports that match - a given criteria (like a domain, but for import data). -* Matching by VAT for ``res.partner`` records will only work when the partner - has no contacts, because otherwise Odoo reflects the parent company's VAT in - the contact, and that results in multiple matches. Fixing the above point - should make this work. +* Add a setting to throw an error when multiple matches are found, instead of + falling back to creation of new record. Bug Tracker =========== diff --git a/base_import_match/__openerp__.py b/base_import_match/__openerp__.py index 4f488895a..83471d213 100644 --- a/base_import_match/__openerp__.py +++ b/base_import_match/__openerp__.py @@ -16,6 +16,8 @@ "base_import", ], "data": [ + "security/ir.model.access.csv", + "data/base_import_match.yml", "views/base_import_match_view.xml", ], "demo": [ diff --git a/base_import_match/data/base_import_match.yml b/base_import_match/data/base_import_match.yml new file mode 100644 index 000000000..5f49c911c --- /dev/null +++ b/base_import_match/data/base_import_match.yml @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +- !context {noupdate: True} + +# Match partners by VAT when is_company is True +- !record {id: res_partner_vat, model: base_import.match}: + model_id: base.model_res_partner + sequence: 10 + +- !record {id: res_partner_vat_vat, model: base_import.match.field}: + match_id: res_partner_vat + field_id: base.field_res_partner_vat + +- !record {id: res_partner_vat_is_company, model: base_import.match.field}: + match_id: res_partner_vat + field_id: base.field_res_partner_is_company + conditional: True + imported_value: "True" + +# Match users by login +- !record {id: res_users_login, model: base_import.match}: + model_id: base.model_res_users + sequence: 50 + +- !record {id: res_users_login_login, model: base_import.match.field}: + match_id: res_users_login + field_id: base.field_res_users_login diff --git a/base_import_match/demo/base_import_match.yml b/base_import_match/demo/base_import_match.yml index 2d6e14be6..110e993ee 100644 --- a/base_import_match/demo/base_import_match.yml +++ b/base_import_match/demo/base_import_match.yml @@ -4,34 +4,37 @@ - !context {noupdate: True} -- !record {id: res_partner_vat, model: base_import.match}: - model_id: base.model_res_partner - sequence: 10 - field_ids: - - base.field_res_partner_vat - +# Match partners by name, parent_id and is_company - !record {id: res_partner_parent_name_is_company, model: base_import.match}: model_id: base.model_res_partner sequence: 20 - field_ids: - - base.field_res_partner_name - - base.field_res_partner_parent_id - - base.field_res_partner_is_company +- !record {id: res_partner_parent_name_is_company_name, model: base_import.match.field}: + match_id: res_partner_parent_name_is_company + field_id: base.field_res_partner_name + +- !record {id: res_partner_parent_name_is_company_parent, model: base_import.match.field}: + match_id: res_partner_parent_name_is_company + field_id: base.field_res_partner_parent_id + +- !record {id: res_partner_parent_name_is_company_is_company, model: base_import.match.field}: + match_id: res_partner_parent_name_is_company + field_id: base.field_res_partner_is_company + +# Match partner by email - !record {id: res_partner_email, model: base_import.match}: model_id: base.model_res_partner sequence: 30 - field_ids: - - base.field_res_partner_email +- !record {id: res_partner_email_email, model: base_import.match.field}: + match_id: res_partner_email + field_id: base.field_res_partner_email + +# Match partner by name - !record {id: res_partner_name, model: base_import.match}: model_id: base.model_res_partner sequence: 40 - field_ids: - - base.field_res_partner_name - -- !record {id: res_users_login, model: base_import.match}: - model_id: base.model_res_users - sequence: 50 - field_ids: - - base.field_res_users_login + +- !record {id: res_partner_name_name, model: base_import.match.field}: + match_id: res_partner_name + field_id: base.field_res_partner_name diff --git a/base_import_match/models/base_import.py b/base_import_match/models/base_import.py index eb97e5edc..cf9bd95f8 100644 --- a/base_import_match/models/base_import.py +++ b/base_import_match/models/base_import.py @@ -3,7 +3,6 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). from openerp import api, fields, models -from openerp.exceptions import except_orm as ValueError # TODO remove in v9 from openerp import SUPERUSER_ID # TODO remove in v10 @@ -11,9 +10,6 @@ class BaseImportMatch(models.Model): _name = "base_import.match" _description = "Deduplicate settings prior to CSV imports." _order = "sequence, name" - _sql_constraints = [ - ("name_unique", "UNIQUE(name)", "Duplicated match!"), - ] name = fields.Char( compute="_compute_name", @@ -31,17 +27,17 @@ class BaseImportMatch(models.Model): related="model_id.model", store=True, index=True) - field_ids = fields.Many2many( - "ir.model.fields", + field_ids = fields.One2many( + comodel_name="base_import.match.field", + inverse_name="match_id", string="Fields", required=True, - domain="[('model_id', '=', model_id)]", help="Fields that will define an unique key.") @api.multi @api.onchange("model_id") def _onchange_model_id(self): - self.field_ids = False + self.field_ids.unlink() @api.model def create(self, vals): @@ -89,7 +85,70 @@ class BaseImportMatch(models.Model): for s in self: s.name = "{}: {}".format( s.model_id.display_name, - " + ".join(s.field_ids.mapped("display_name"))) + " + ".join( + s.field_ids.mapped( + lambda r: ( + str(r.field_id.name) + + (" ({})".format(r.imported_value) + if r.conditional else ""))))) + + @api.model + def _match_find(self, model, converted_row, imported_row): + """Find a update target for the given row. + + This will traverse by order all match rules that can be used with the + imported data, and return a match for the first rule that returns a + single result. + + :param openerp.models.Model model: + Model object that is being imported. + + :param dict converted_row: + Row converted to Odoo api format, like the 3rd value that + :meth:`openerp.models.Model._convert_records` returns. + + :param dict imported_row: + Row as it is being imported, in format:: + + { + "field_name": "string value", + "other_field": "True", + ... + } + + :return openerp.models.Model: + Return a dataset with one single match if it was found, or an + empty dataset if none or multiple matches were found. + """ + # Get usable rules to perform matches + usable = self._usable_for_load(model._name, converted_row.keys()) + + # Traverse usable combinations + for combination in usable: + combination_valid = True + domain = list() + + for field in combination.field_ids: + # Check imported value if it is a conditional field + if field.conditional: + # Invalid combinations are skipped + if imported_row[field.name] != field.imported_value: + combination_valid = False + break + + domain.append((field.name, "=", converted_row[field.name])) + + if not combination_valid: + continue + + match = model.search(domain) + + # When a single match is found, stop searching + if len(match) == 1: + return match + + # Return an empty match if none or multiple was found + return model @api.model def _load_wrapper(self): @@ -105,53 +164,37 @@ class BaseImportMatch(models.Model): """ newdata = list() + # Data conversion to ORM format + import_fields = map(models.fix_import_export_id_paths, fields) + converted_data = self._convert_records( + self._extract_records(import_fields, data)) + # Mock Odoo to believe the user is importing the ID field if "id" not in fields: fields.append("id") + import_fields.append(["id"]) - # Needed to work with relational fields - clean_fields = [ - models.fix_import_export_id_paths(f)[0] for f in fields] + # Needed to match with converted data field names + clean_fields = [f[0] for f in import_fields] - # Get usable rules to perform matches - usable = self.env["base_import.match"]._usable_for_load( - self._name, clean_fields) - - for row in (dict(zip(clean_fields, r)) for r in data): - # All rows need an ID - if "id" not in row: - row["id"] = u"" - - # Skip rows with ID, they do not need all this - elif row["id"]: - continue - - # Store records that match a combination + for dbid, xmlid, record, info in converted_data: + row = dict(zip(clean_fields, data[info["record"]])) match = self - for combination in usable: - match |= self.search( - [(field.name, "=", row[field.name]) - for field in combination.field_ids]) - - # When a single match is found, stop searching - if len(match) != 1: - break - # Only one record should have been found - try: - match.ensure_one() - - # You hit this because... - # a. No match. Odoo must create a new record. - # b. Multiple matches. No way to know which is the right - # one, so we let Odoo create a new record or raise - # the corresponding exception. - # In any case, we must do nothing. - except ValueError: - continue - - # Give a valid XMLID to this row - row["id"] = match._BaseModel__export_xml_id() + if xmlid: + # Skip rows with ID, they do not need all this + row["id"] = xmlid + elif dbid: + # Find the xmlid for this dbid + match = self.browse(dbid) + else: + # Store records that match a combination + match = self.env["base_import.match"]._match_find( + self, record, row) + + # Give a valid XMLID to this row if a match was found + row["id"] = (match._BaseModel__export_xml_id() + if match else row.get("id", u"")) # Store the modified row, in the same order as fields newdata.append(tuple(row[f] for f in clean_fields)) @@ -218,3 +261,37 @@ class BaseImportMatch(models.Model): result += record return result + + +class BaseImportMatchField(models.Model): + _name = "base_import.match.field" + _description = "Field import match definition" + + name = fields.Char( + related="field_id.name") + field_id = fields.Many2one( + comodel_name="ir.model.fields", + string="Field", + required=True, + ondelete="cascade", + domain="[('model_id', '=', model_id)]", + help="Field that will be part of an unique key.") + match_id = fields.Many2one( + comodel_name="base_import.match", + string="Match", + required=True) + model_id = fields.Many2one( + related="match_id.model_id") + conditional = fields.Boolean( + help="Enable if you want to use this field only in some conditions.") + imported_value = fields.Char( + help="If the imported value is not this, the whole matching rule will " + "be discarded. Be careful, this data is always treated as a " + "string, and comparison is case-sensitive so if you set 'True', " + "it will NOT match '1' nor 'true', only EXACTLY 'True'.") + + @api.multi + @api.onchange("field_id", "match_id", "conditional", "imported_value") + def _onchange_match_id_name(self): + """Update match name.""" + self.mapped("match_id")._compute_name() diff --git a/base_import_match/security/ir.model.access.csv b/base_import_match/security/ir.model.access.csv new file mode 100644 index 000000000..6ffd90137 --- /dev/null +++ b/base_import_match/security/ir.model.access.csv @@ -0,0 +1,5 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_base_import_match,Access base_import.match,model_base_import_match,base.group_user,1,0,0,0 +access_base_import_match_field,Access base_import.match.field,model_base_import_match_field,base.group_user,1,0,0,0 +write_base_import_match,Write base_import.match,model_base_import_match,base.group_system,1,1,1,1 +write_base_import_match_field,Write base_import.match.field,model_base_import_match_field,base.group_system,1,1,1,1 diff --git a/base_import_match/tests/import_data/res_partner_vat.csv b/base_import_match/tests/import_data/res_partner_vat.csv index bbdaa3e59..acc3188cf 100644 --- a/base_import_match/tests/import_data/res_partner_vat.csv +++ b/base_import_match/tests/import_data/res_partner_vat.csv @@ -1,2 +1,2 @@ -name,vat -Federal Changed,BE0477472701 +name,vat,is_company +Agrolait Changed,BE0477472701,True diff --git a/base_import_match/tests/test_import.py b/base_import_match/tests/test_import.py index 9f65ebe57..5c47663d7 100644 --- a/base_import_match/tests/test_import.py +++ b/base_import_match/tests/test_import.py @@ -27,12 +27,12 @@ class ImportCase(TransactionCase): def test_res_partner_vat(self): """Change name based on VAT.""" - federal = self.env.ref("base.res_partner_26") - federal.vat = "BE0477472701" + agrolait = self.env.ref("base.res_partner_2") + agrolait.vat = "BE0477472701" record = self._base_import_record("res.partner", "res_partner_vat") - record.do(["name", "vat"], OPTIONS) - federal.env.invalidate_all() - self.assertEqual(federal.name, "Federal Changed") + record.do(["name", "vat", "is_company"], OPTIONS) + agrolait.env.invalidate_all() + self.assertEqual(agrolait.name, "Agrolait Changed") def test_res_partner_parent_name_is_company(self): """Change email based on parent_id, name and is_company.""" diff --git a/base_import_match/views/base_import_match_view.xml b/base_import_match/views/base_import_match_view.xml index 7eeb26e5f..06e62d491 100644 --- a/base_import_match/views/base_import_match_view.xml +++ b/base_import_match/views/base_import_match_view.xml @@ -16,12 +16,22 @@ - + + + + + + + + +