diff --git a/base_import_match/README.rst b/base_import_match/README.rst new file mode 100644 index 000000000..6066502a6 --- /dev/null +++ b/base_import_match/README.rst @@ -0,0 +1,109 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 + +================= +Base Import Match +================= + +By default, when importing data (like CSV import) with the ``base_import`` +module, Odoo follows this rule: + +#. If you import the XMLID of a record, make an **update**. +#. If you do not, **create** a new record. + +This module allows you to set additional rules to match if a given import is an +update or a new record. + +This is useful when you need to sync heterogeneous databases, and the field you +use to match records in those databases with Odoo's is not the XMLID but the +name, VAT, email, etc. + +After installing this module, the import logic will be changed to: + +#. If you import the XMLID of a record, make an **update**. +#. If you do not: + #. If there are import match rules for the model you are importing: + #. Discard the rules that require fields you are not importing. + #. Traverse the remaining rules one by one in order to find a match in + the database. + #. If one match is found: + #. Stop traversing the rest of valid rules. + #. **Update** that record. + #. If zero or multiple matches are found: + #. Continue with the next rule. + #. If all rules are exhausted and no single match is found: + #. **Create** a new record. + #. If there are no match rules for your model: + #. **Create** a new record. + +Configuration +============= + +To configure this module, you need to: + +#. Go to *Settings > Technical > Database Structure > Import Match*. +#. *Create*. +#. Choose a *Model*. +#. Choose the *Fields* that conform an unique key in that model. +#. *Save*. + +In that list view, you can sort rules by drag and drop. + +Usage +===== + +To use this module, you need to: + +#. Follow steps in **Configuration** section above. +#. Go to any list view. +#. Press *Import* and follow the import procedure as usual. + +.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas + :alt: Try me on Runbot + :target: https://runbot.odoo-community.org/runbot/149/8.0 + +Roadmap / Known Issues +====================== + +* Add a filter to let you apply some rules only to incoming imports that match + a given criteria (like a domain, but for import data). +* Matching by VAT for ``res.partner`` records will only work when the partner + has no contacts, because otherwise Odoo reflects the parent company's VAT in + the contact, and that results in multiple matches. Fixing the above point + should make this work. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues +`_. In case of trouble, please +check there if your issue has already been reported. If you spotted it first, +help us smashing it by providing a detailed and welcomed `feedback +`_. + +Credits +======= + +Contributors +------------ + +* Jairo Llopis + +Maintainer +---------- + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit https://odoo-community.org. diff --git a/base_import_match/__init__.py b/base_import_match/__init__.py new file mode 100644 index 000000000..09356048f --- /dev/null +++ b/base_import_match/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from . import models diff --git a/base_import_match/__openerp__.py b/base_import_match/__openerp__.py new file mode 100644 index 000000000..4f488895a --- /dev/null +++ b/base_import_match/__openerp__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +{ + "name": "Base Import Match", + "summary": "Try to avoid duplicates before importing", + "version": "8.0.1.0.0", + "category": "Tools", + "website": "https://grupoesoc.es", + "author": "Grupo ESOC Ingeniería de Servicios, " + "Odoo Community Association (OCA)", + "license": "AGPL-3", + "application": False, + "installable": True, + "depends": [ + "base_import", + ], + "data": [ + "views/base_import_match_view.xml", + ], + "demo": [ + "demo/base_import_match.yml", + ], +} diff --git a/base_import_match/demo/base_import_match.yml b/base_import_match/demo/base_import_match.yml new file mode 100644 index 000000000..2d6e14be6 --- /dev/null +++ b/base_import_match/demo/base_import_match.yml @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +- !context {noupdate: True} + +- !record {id: res_partner_vat, model: base_import.match}: + model_id: base.model_res_partner + sequence: 10 + field_ids: + - base.field_res_partner_vat + +- !record {id: res_partner_parent_name_is_company, model: base_import.match}: + model_id: base.model_res_partner + sequence: 20 + field_ids: + - base.field_res_partner_name + - base.field_res_partner_parent_id + - base.field_res_partner_is_company + +- !record {id: res_partner_email, model: base_import.match}: + model_id: base.model_res_partner + sequence: 30 + field_ids: + - base.field_res_partner_email + +- !record {id: res_partner_name, model: base_import.match}: + model_id: base.model_res_partner + sequence: 40 + field_ids: + - base.field_res_partner_name + +- !record {id: res_users_login, model: base_import.match}: + model_id: base.model_res_users + sequence: 50 + field_ids: + - base.field_res_users_login diff --git a/base_import_match/i18n/es.po b/base_import_match/i18n/es.po new file mode 100644 index 000000000..6f7fbc6c6 --- /dev/null +++ b/base_import_match/i18n/es.po @@ -0,0 +1,83 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * base_import_match +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 8.0\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2016-05-11 12:24+0200\n" +"PO-Revision-Date: 2016-05-11 12:26+0200\n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Poedit 1.8.7.1\n" +"Last-Translator: Jairo Llopis \n" +"Language: es_ES\n" + +#. module: base_import_match +#: model:ir.model,name:base_import_match.model_base_import_match +msgid "Deduplicate settings prior to CSV imports." +msgstr "Configuración para deduplicar antes de importar CSV." + +#. module: base_import_match +#: field:base_import.match,display_name:0 +msgid "Display Name" +msgstr "Nombre a mostrar" + +#. module: base_import_match +#: sql_constraint:base_import.match:0 +msgid "Duplicated match!" +msgstr "¡Coincidencia duplicada!" + +#. module: base_import_match +#: field:base_import.match,field_ids:0 +msgid "Fields" +msgstr "Campos" + +#. module: base_import_match +#: help:base_import.match,field_ids:0 +msgid "Fields that will define an unique key." +msgstr "Campos que definirán una clave única." + +#. module: base_import_match +#: view:base_import.match:base_import_match.match_search_view +msgid "Group By" +msgstr "Agrupar por" + +#. module: base_import_match +#: view:base_import.match:base_import_match.match_form_view +#: view:base_import.match:base_import_match.match_search_view +#: view:base_import.match:base_import_match.match_tree_view +#: model:ir.actions.act_window,name:base_import_match.match_action +#: model:ir.ui.menu,name:base_import_match.match_menu +msgid "Import Match" +msgstr "Coincidencia de importación" + +#. module: base_import_match +#: help:base_import.match,model_id:0 +msgid "In this model you will apply the match." +msgstr "En este modelo se aplicará la coincidencia." + +#. module: base_import_match +#: field:base_import.match,__last_update:0 +msgid "Last Modified on" +msgstr "Última actualización por" + +#. module: base_import_match +#: view:base_import.match:base_import_match.match_search_view +#: field:base_import.match,model_id:0 field:base_import.match,model_name:0 +msgid "Model" +msgstr "Modelo" + +#. module: base_import_match +#: field:base_import.match,name:0 +msgid "Name" +msgstr "Nombre" + +#. module: base_import_match +#: field:base_import.match,sequence:0 +msgid "Sequence" +msgstr "Secuencia" diff --git a/base_import_match/models/__init__.py b/base_import_match/models/__init__.py new file mode 100644 index 000000000..ec83d2a4b --- /dev/null +++ b/base_import_match/models/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from . import base_import diff --git a/base_import_match/models/base_import.py b/base_import_match/models/base_import.py new file mode 100644 index 000000000..eb97e5edc --- /dev/null +++ b/base_import_match/models/base_import.py @@ -0,0 +1,220 @@ +# -*- coding: utf-8 -*- +# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from openerp import api, fields, models +from openerp.exceptions import except_orm as ValueError # TODO remove in v9 +from openerp import SUPERUSER_ID # TODO remove in v10 + + +class BaseImportMatch(models.Model): + _name = "base_import.match" + _description = "Deduplicate settings prior to CSV imports." + _order = "sequence, name" + _sql_constraints = [ + ("name_unique", "UNIQUE(name)", "Duplicated match!"), + ] + + name = fields.Char( + compute="_compute_name", + store=True, + index=True) + sequence = fields.Integer(index=True) + model_id = fields.Many2one( + "ir.model", + "Model", + required=True, + ondelete="cascade", + domain=[("osv_memory", "=", False)], + help="In this model you will apply the match.") + model_name = fields.Char( + related="model_id.model", + store=True, + index=True) + field_ids = fields.Many2many( + "ir.model.fields", + string="Fields", + required=True, + domain="[('model_id', '=', model_id)]", + help="Fields that will define an unique key.") + + @api.multi + @api.onchange("model_id") + def _onchange_model_id(self): + self.field_ids = False + + @api.model + def create(self, vals): + """Wrap the model after creation.""" + result = super(BaseImportMatch, self).create(vals) + self._load_autopatch(result.model_name) + return result + + @api.multi + def unlink(self): + """Unwrap the model after deletion.""" + models = set(self.mapped("model_name")) + result = super(BaseImportMatch, self).unlink() + for model in models: + self._load_autopatch(model) + return result + + @api.multi + def write(self, vals): + """Wrap the model after writing.""" + result = super(BaseImportMatch, self).write(vals) + + if "model_id" in vals or "model_name" in vals: + for s in self: + self._load_autopatch(s.model_name) + + return result + + # TODO convert to @api.model_cr in v10 + def _register_hook(self, cr): + """Autopatch on init.""" + models = set( + self.browse( + cr, + SUPERUSER_ID, + self.search(cr, SUPERUSER_ID, list())) + .mapped("model_name")) + for model in models: + self._load_autopatch(cr, SUPERUSER_ID, model) + + @api.multi + @api.depends("model_id", "field_ids") + def _compute_name(self): + """Automatic self-descriptive name for the setting records.""" + for s in self: + s.name = "{}: {}".format( + s.model_id.display_name, + " + ".join(s.field_ids.mapped("display_name"))) + + @api.model + def _load_wrapper(self): + """Create a new load patch method.""" + @api.model + def wrapper(self, fields, data): + """Try to identify rows by other pseudo-unique keys. + + It searches for rows that have no XMLID specified, and gives them + one if any :attr:`~.field_ids` combination is found. With a valid + XMLID in place, Odoo will understand that it must *update* the + record instead of *creating* a new one. + """ + newdata = list() + + # Mock Odoo to believe the user is importing the ID field + if "id" not in fields: + fields.append("id") + + # Needed to work with relational fields + clean_fields = [ + models.fix_import_export_id_paths(f)[0] for f in fields] + + # Get usable rules to perform matches + usable = self.env["base_import.match"]._usable_for_load( + self._name, clean_fields) + + for row in (dict(zip(clean_fields, r)) for r in data): + # All rows need an ID + if "id" not in row: + row["id"] = u"" + + # Skip rows with ID, they do not need all this + elif row["id"]: + continue + + # Store records that match a combination + match = self + for combination in usable: + match |= self.search( + [(field.name, "=", row[field.name]) + for field in combination.field_ids]) + + # When a single match is found, stop searching + if len(match) != 1: + break + + # Only one record should have been found + try: + match.ensure_one() + + # You hit this because... + # a. No match. Odoo must create a new record. + # b. Multiple matches. No way to know which is the right + # one, so we let Odoo create a new record or raise + # the corresponding exception. + # In any case, we must do nothing. + except ValueError: + continue + + # Give a valid XMLID to this row + row["id"] = match._BaseModel__export_xml_id() + + # Store the modified row, in the same order as fields + newdata.append(tuple(row[f] for f in clean_fields)) + + # Leave the rest to Odoo itself + del data + return wrapper.origin(self, fields, newdata) + + # Flag to avoid confusions with other possible wrappers + wrapper.__base_import_match = True + + return wrapper + + @api.model + def _load_autopatch(self, model_name): + """[Un]apply patch automatically.""" + self._load_unpatch(model_name) + if self.search([("model_name", "=", model_name)]): + self._load_patch(model_name) + + @api.model + def _load_patch(self, model_name): + """Apply patch for :param:`model_name`'s load method. + + :param str model_name: + Model technical name, such as ``res.partner``. + """ + self.env[model_name]._patch_method( + "load", self._load_wrapper()) + + @api.model + def _load_unpatch(self, model_name): + """Apply patch for :param:`model_name`'s load method. + + :param str model_name: + Model technical name, such as ``res.partner``. + """ + model = self.env[model_name] + + # Unapply patch only if there is one + try: + if model.load.__base_import_match: + model._revert_method("load") + except AttributeError: + pass + + @api.model + def _usable_for_load(self, model_name, fields): + """Return a set of elements usable for calling ``load()``. + + :param str model_name: + Technical name of the model where you are loading data. + E.g. ``res.partner``. + + :param list(str|bool) fields: + List of field names being imported. + """ + result = self + available = self.search([("model_name", "=", model_name)]) + + # Use only criteria with all required fields to match + for record in available: + if all(f.name in fields for f in record.field_ids): + result += record + + return result diff --git a/base_import_match/static/description/icon.png b/base_import_match/static/description/icon.png new file mode 100644 index 000000000..3a0328b51 Binary files /dev/null and b/base_import_match/static/description/icon.png differ diff --git a/base_import_match/tests/__init__.py b/base_import_match/tests/__init__.py new file mode 100644 index 000000000..ce1b8ba33 --- /dev/null +++ b/base_import_match/tests/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from . import test_import diff --git a/base_import_match/tests/import_data/res_partner_email.csv b/base_import_match/tests/import_data/res_partner_email.csv new file mode 100644 index 000000000..983b91133 --- /dev/null +++ b/base_import_match/tests/import_data/res_partner_email.csv @@ -0,0 +1,2 @@ +email,name +michel.fletcher@agrolait.example.com,Michel Fletcher Changed diff --git a/base_import_match/tests/import_data/res_partner_name.csv b/base_import_match/tests/import_data/res_partner_name.csv new file mode 100644 index 000000000..c1a109590 --- /dev/null +++ b/base_import_match/tests/import_data/res_partner_name.csv @@ -0,0 +1,2 @@ +function,name +Function Changed,Michel Fletcher diff --git a/base_import_match/tests/import_data/res_partner_parent_name_is_company.csv b/base_import_match/tests/import_data/res_partner_parent_name_is_company.csv new file mode 100644 index 000000000..2d433d94e --- /dev/null +++ b/base_import_match/tests/import_data/res_partner_parent_name_is_company.csv @@ -0,0 +1,2 @@ +name,is_company,parent_id/id,email +Michel Fletcher,False,base.res_partner_2,changed@agrolait.example.com diff --git a/base_import_match/tests/import_data/res_partner_vat.csv b/base_import_match/tests/import_data/res_partner_vat.csv new file mode 100644 index 000000000..bbdaa3e59 --- /dev/null +++ b/base_import_match/tests/import_data/res_partner_vat.csv @@ -0,0 +1,2 @@ +name,vat +Federal Changed,BE0477472701 diff --git a/base_import_match/tests/import_data/res_users_login.csv b/base_import_match/tests/import_data/res_users_login.csv new file mode 100644 index 000000000..e91cccad2 --- /dev/null +++ b/base_import_match/tests/import_data/res_users_login.csv @@ -0,0 +1,2 @@ +login,name +demo,Demo User Changed diff --git a/base_import_match/tests/test_import.py b/base_import_match/tests/test_import.py new file mode 100644 index 000000000..9f65ebe57 --- /dev/null +++ b/base_import_match/tests/test_import.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# © 2016 Grupo ESOC Ingeniería de Servicios, S.L.U. - Jairo Llopis +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from os import path +from openerp.tests.common import TransactionCase + + +PATH = path.join(path.dirname(__file__), "import_data", "%s.csv") +OPTIONS = { + "headers": True, + "quoting": '"', + "separator": ",", +} + + +class ImportCase(TransactionCase): + def _base_import_record(self, res_model, file_name): + """Create and return a ``base_import.import`` record.""" + with open(PATH % file_name) as demo_file: + return self.env["base_import.import"].create({ + "res_model": res_model, + "file": demo_file.read(), + "file_name": "%s.csv" % file_name, + "file_type": "csv", + }) + + def test_res_partner_vat(self): + """Change name based on VAT.""" + federal = self.env.ref("base.res_partner_26") + federal.vat = "BE0477472701" + record = self._base_import_record("res.partner", "res_partner_vat") + record.do(["name", "vat"], OPTIONS) + federal.env.invalidate_all() + self.assertEqual(federal.name, "Federal Changed") + + def test_res_partner_parent_name_is_company(self): + """Change email based on parent_id, name and is_company.""" + record = self._base_import_record( + "res.partner", "res_partner_parent_name_is_company") + record.do(["name", "is_company", "parent_id/id", "email"], OPTIONS) + self.assertEqual( + self.env.ref("base.res_partner_address_4").email, + "changed@agrolait.example.com") + + def test_res_partner_email(self): + """Change name based on email.""" + record = self._base_import_record("res.partner", "res_partner_email") + record.do(["email", "name"], OPTIONS) + self.assertEqual( + self.env.ref("base.res_partner_address_4").name, + "Michel Fletcher Changed") + + def test_res_partner_name(self): + """Change function based on name.""" + record = self._base_import_record("res.partner", "res_partner_name") + record.do(["function", "name"], OPTIONS) + self.assertEqual( + self.env.ref("base.res_partner_address_4").function, + "Function Changed") + + def test_res_users_login(self): + """Change name based on login.""" + record = self._base_import_record("res.users", "res_users_login") + record.do(["login", "name"], OPTIONS) + self.assertEqual( + self.env.ref("base.user_demo").name, + "Demo User Changed") diff --git a/base_import_match/views/base_import_match_view.xml b/base_import_match/views/base_import_match_view.xml new file mode 100644 index 000000000..7eeb26e5f --- /dev/null +++ b/base_import_match/views/base_import_match_view.xml @@ -0,0 +1,70 @@ + + + + + + + + Import match form view + base_import.match + +
+ +

+ +

+ + + + + + +
+ + + + + Import match tree view + base_import.match + + + + + + + + + + Import match search view + base_import.match + + + + + + + + + + + + + + + + + + +