From ecef4afb33bfd339986248b1075c8792592b9057 Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Sun, 17 Sep 2017 22:18:29 +0200 Subject: [PATCH 01/21] [ADD] base_import_odoo --- .travis.yml | 1 + base_import_odoo/README.rst | 94 ++++ base_import_odoo/__init__.py | 4 + base_import_odoo/__openerp__.py | 29 ++ .../demo/import_odoo_database.xml | 11 + .../demo/import_odoo_database_field.xml | 23 + .../demo/import_odoo_database_model.xml | 17 + base_import_odoo/demo/res_users.xml | 9 + base_import_odoo/models/__init__.py | 7 + .../models/import_odoo_database.py | 472 ++++++++++++++++++ .../models/import_odoo_database_field.py | 35 ++ .../models/import_odoo_database_model.py | 20 + base_import_odoo/models/ir_model_data.py | 13 + base_import_odoo/security/ir.model.access.csv | 4 + base_import_odoo/static/description/icon.png | Bin 0 -> 9455 bytes base_import_odoo/tests/__init__.py | 4 + .../tests/test_base_import_odoo.py | 68 +++ .../views/import_odoo_database.xml | 89 ++++ base_import_odoo/views/menu.xml | 12 + 19 files changed, 912 insertions(+) create mode 100644 base_import_odoo/README.rst create mode 100644 base_import_odoo/__init__.py create mode 100644 base_import_odoo/__openerp__.py create mode 100644 base_import_odoo/demo/import_odoo_database.xml create mode 100644 base_import_odoo/demo/import_odoo_database_field.xml create mode 100644 base_import_odoo/demo/import_odoo_database_model.xml create mode 100644 base_import_odoo/demo/res_users.xml create mode 100644 base_import_odoo/models/__init__.py create mode 100644 base_import_odoo/models/import_odoo_database.py create mode 100644 base_import_odoo/models/import_odoo_database_field.py create mode 100644 base_import_odoo/models/import_odoo_database_model.py create mode 100644 base_import_odoo/models/ir_model_data.py create mode 100644 base_import_odoo/security/ir.model.access.csv create mode 100644 base_import_odoo/static/description/icon.png create mode 100644 base_import_odoo/tests/__init__.py create mode 100644 base_import_odoo/tests/test_base_import_odoo.py create mode 100644 base_import_odoo/views/import_odoo_database.xml create mode 100644 base_import_odoo/views/menu.xml diff --git a/.travis.yml b/.travis.yml index d3e355a6e..50f6081b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,7 @@ install: - export PATH=${HOME}/maintainer-quality-tools/travis:${PATH} - travis_install_nightly - printf '[options]\n\nrunning_env = dev\n' > ${HOME}/.openerp_serverrc + - pip install erppeek - ln -s ${TRAVIS_BUILD_DIR}/server_environment_files_sample ${TRAVIS_BUILD_DIR}/server_environment_files script: - travis_run_tests diff --git a/base_import_odoo/README.rst b/base_import_odoo/README.rst new file mode 100644 index 000000000..73c0fee64 --- /dev/null +++ b/base_import_odoo/README.rst @@ -0,0 +1,94 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 + +================================ +Import from remote Odoo database +================================ + +This module was written to import data from another Odoo database. The idea is that you define which models to import from the other database, and add eventual mappings for records you don't want to import. + +Use cases +========= + +- merging databases +- one way sync (needs a bit polishing) +- aggregating management data from distributed systems + + +Configuration +============= + +Go to Settings / Remote Odoo import / Import configurations and create a configuration. + +After filling in your credentials, select models you want to import from the remote database. If you only want to import a subset of the records, add an appropriate domain. + +The import will copy records of all models listed, and handle links to records of models which are not imported depending on the existing field mappings. Field mappings to local records also are a stopping condition. Without those, the import will have to create some record for all required x2x fields, which you probably don't want. + +Probably you'll want to map records of model `res.company`, and at least the admin user. + +The module doesn't import one2many fields, if you want to have those, add the model the field in question points to to the list of imported models, possibly with a domain. + +If you don't fill in a remote ID, the addon will use the configured local ID for every record of the model (this way, you can for example map all users in the remote system to some import user in the current system). + +For fields that have a uniqueness constraint (like `res.users#login`), set the flag `unique`, then the import will generate a unique value for this field. + +Usage +===== + +To use this module, you need to: + +#. go to an import configuration and hit the button ``Run import`` +#. be patient, this creates a cronjob which will start up to a minutes afterwards +#. reload the form, as soon as the cronjob runs you'll see a field ``Progress`` that lets you inspect what was imported already +#. note that the cronjob also resets the password as soon as it has read it. So for a subsequent import, you'll have to fill it in again +#. running an import a second time won't duplicate data, it should recognize records imported earlier and just update them + +.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas + :alt: Try me on Runbot + :target: https://runbot.odoo-community.org/runbot/149/8.0 + +Known issues / Roadmap +====================== + +* Yes of course this duplicates a lot of connector functionality. Rewrite this with the connector framework, probably collaborate with https://github.com/OCA/connector-odoo2odoo +* Do something with workflows +* Probably it's safer and faster to disable recomputation during import, and recompute all fields afterwards + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues +`_. In case of trouble, please +check there if your issue has already been reported. If you spotted it first, +help us smashing it by providing a detailed and welcomed feedback. + +Credits +======= + +Images +------ + +* Odoo Community Association: `Icon `_. + +Contributors +------------ + +* Holger Brunn + +Do not contact contributors directly about help with questions or problems concerning this addon, but use the `community mailing list `_ or the `appropriate specialized mailinglist `_ for help, and the bug tracker linked in `Bug Tracker`_ above for technical issues. + +Maintainer +---------- + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit https://odoo-community.org. diff --git a/base_import_odoo/__init__.py b/base_import_odoo/__init__.py new file mode 100644 index 000000000..86cb334c3 --- /dev/null +++ b/base_import_odoo/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from . import models diff --git a/base_import_odoo/__openerp__.py b/base_import_odoo/__openerp__.py new file mode 100644 index 000000000..baaba59e5 --- /dev/null +++ b/base_import_odoo/__openerp__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +{ + "name": "Import from Odoo", + "version": "8.0.1.0.0", + "author": "Therp BV,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Tools", + "summary": "Import records from another Odoo instance", + "depends": [ + 'base', + ], + "demo": [ + "demo/res_users.xml", + "demo/import_odoo_database.xml", + "demo/import_odoo_database_field.xml", + "demo/import_odoo_database_model.xml", + ], + "data": [ + "security/ir.model.access.csv", + "views/import_odoo_database.xml", + "views/menu.xml", + ], + "installable": True, + "external_dependencies": { + "python": ['erppeek'], + }, +} diff --git a/base_import_odoo/demo/import_odoo_database.xml b/base_import_odoo/demo/import_odoo_database.xml new file mode 100644 index 000000000..f1f9bb59d --- /dev/null +++ b/base_import_odoo/demo/import_odoo_database.xml @@ -0,0 +1,11 @@ + + + + + http://localhost:8069 + demodb + admin + + + + diff --git a/base_import_odoo/demo/import_odoo_database_field.xml b/base_import_odoo/demo/import_odoo_database_field.xml new file mode 100644 index 000000000..e47c6d070 --- /dev/null +++ b/base_import_odoo/demo/import_odoo_database_field.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/base_import_odoo/demo/import_odoo_database_model.xml b/base_import_odoo/demo/import_odoo_database_model.xml new file mode 100644 index 000000000..c9c856766 --- /dev/null +++ b/base_import_odoo/demo/import_odoo_database_model.xml @@ -0,0 +1,17 @@ + + + + + 1 + + + [(1, '=', 1)] + + + 2 + + + [(1, '=', 1)] + + + diff --git a/base_import_odoo/demo/res_users.xml b/base_import_odoo/demo/res_users.xml new file mode 100644 index 000000000..152e5cbd3 --- /dev/null +++ b/base_import_odoo/demo/res_users.xml @@ -0,0 +1,9 @@ + + + + + Mapped admin + mapped_admin + + + diff --git a/base_import_odoo/models/__init__.py b/base_import_odoo/models/__init__.py new file mode 100644 index 000000000..2fd595611 --- /dev/null +++ b/base_import_odoo/models/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from . import ir_model_data +from . import import_odoo_database +from . import import_odoo_database_model +from . import import_odoo_database_field diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py new file mode 100644 index 000000000..8a15b73f4 --- /dev/null +++ b/base_import_odoo/models/import_odoo_database.py @@ -0,0 +1,472 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +try: + from erppeek import Client +except: + pass +import psycopg2 +import traceback +from openerp import _, api, exceptions, fields, models, tools +from collections import namedtuple + + +import_context_tuple = namedtuple( + 'import_context', [ + 'remote', 'model_line', 'ids', 'idmap', 'dummies', 'dummy_instances', + 'to_delete', 'field_context', + ] +) + + +class ImportContext(import_context_tuple): + def with_field_context(self, *args): + return ImportContext(*(self[:-1] + (field_context(*args),))) + + +field_context = namedtuple( + 'field_context', ['record_model', 'field_name', 'record_id'], +) + + +mapping_key = namedtuple('mapping_key', ['model_name', 'remote_id']) + + +dummy_instance = namedtuple( + 'dummy_instance', ['model_name', 'field_name', 'remote_id', 'dummy_id'], +) + + +class ImportOdooDatabase(models.Model): + _name = 'import.odoo.database' + _description = 'An Odoo database to import' + + url = fields.Char(required=True) + database = fields.Char(required=True) + user = fields.Char(default='admin', required=True) + password = fields.Char(default='admin') + import_line_ids = fields.One2many( + 'import.odoo.database.model', 'database_id', string='Import models', + ) + import_field_mappings = fields.One2many( + 'import.odoo.database.field', 'database_id', string='Field mappings', + ) + cronjob_id = fields.Many2one( + 'ir.cron', string='Import job', readonly=True, copy=False, + ) + cronjob_running = fields.Boolean(compute='_compute_cronjob_running') + status_data = fields.Serialized('Status', readonly=True, copy=False) + status_html = fields.Html( + compute='_compute_status_html', readonly=True, sanitize=False, + ) + + @api.multi + def action_import(self): + """Create a cronjob to run the actual import""" + self.ensure_one() + if self.cronjob_id: + return self.cronjob_id.write({ + 'numbercall': 1, + 'doall': True, + 'active': True, + }) + return self.write({ + 'cronjob_id': self._create_cronjob().id, + }) + + @api.multi + def _run_import(self, commit=True, commit_threshold=100): + """Run the import as cronjob, commit often""" + self.ensure_one() + if not self.password: + return + # model name: [ids] + remote_ids = {} + # model name: count + remote_counts = {} + # model name: count + done = {} + # mapping_key: local_id + idmap = {} + # mapping_key: local_id + # this are records created or linked when we need to fill a required + # field, but the local record is not yet created + dummies = {} + # model name: [local_id] + # this happens when we create a dummy we can throw away again + to_delete = {} + # dummy_instance + dummy_instances = [] + remote = self._get_connection() + self.write({'password': False}) + if commit and not tools.config['test_enable']: + # pylint: disable=invalid-commit + self.env.cr.commit() + for model_line in self.import_line_ids: + model = model_line.model_id + remote_ids[model.model] = remote.search( + model.model, + tools.safe_eval(model_line.domain) if model_line.domain else [] + ) + remote_counts[model.model] = len(remote_ids[model.model]) + self.write({ + 'status_data': { + 'counts': remote_counts, + 'ids': remote_ids, + 'error': None, + 'done': {}, + } + }) + if commit and not tools.config['test_enable']: + # pylint: disable=invalid-commit + self.env.cr.commit() + for model_line in self.import_line_ids: + model = self.env[model_line.model_id.model] + done[model._name] = 0 + + for start_index in range( + len(remote_ids[model._name]) / commit_threshold + 1 + ): + index = start_index * commit_threshold + ids = remote_ids[model._name][index:index + commit_threshold] + context = ImportContext( + remote, model_line, ids, idmap, dummies, dummy_instances, + to_delete, field_context(None, None, None), + ) + try: + self._run_import_model(context) + except: + error = traceback.format_exc() + self.env.cr.rollback() + self.write({ + 'status_data': dict(self.status_data, error=error), + }) + # pylint: disable=invalid-commit + self.env.cr.commit() + raise + done[model._name] += len(ids) + self.write({'status_data': dict(self.status_data, done=done)}) + if commit and not tools.config['test_enable']: + # pylint: disable=invalid-commit + self.env.cr.commit() + + @api.multi + def _run_import_model(self, context): + """Import records of a configured model""" + model = self.env[context.model_line.model_id.model] + fields = self._run_import_model_get_fields(context) + for data in context.remote.read( + model._name, context.ids, fields.keys() + ): + self._run_import_get_record(context, model, data) + if (model._name, data['id']) in context.idmap: + # there's a mapping for this record, nothing to do + continue + data = self._run_import_map_values(context, data) + _id = data['id'] + record = self._create_record(context, model, data) + self._run_import_model_cleanup_dummies( + context, model, _id, record.id, + ) + + @api.multi + def _create_record(self, context, model, record): + """Create a record, add an xmlid""" + _id = record.pop('id') + xmlid = '%d-%s-%d' % ( + self.id, model._name.replace('.', '_'), _id, + ) + if self.env.ref('base_import_odoo.%s' % xmlid, False): + new = self.env.ref('base_import_odoo.%s' % xmlid) + new.with_context( + **self._create_record_context(model, record) + ).write(record) + else: + new = model.with_context( + **self._create_record_context(model, record) + ).create(record) + self.env['ir.model.data'].create({ + 'name': xmlid, + 'model': model._name, + 'module': 'base_import_odoo', + 'res_id': new.id, + 'noupdate': True, + 'import_database_id': self.id, + 'import_database_record_id': _id, + }) + context.idmap[mapping_key(model._name, _id)] = new.id + return new + + def _create_record_context(self, model, record): + """Return a context that is used when creating a record""" + context = { + 'tracking_disable': True, + } + if model._name == 'res.users': + context['no_reset_password'] = True + return context + + @api.multi + def _run_import_get_record( + self, context, model, record, create_dummy=True, + ): + """Find the local id of some remote record. Create a dummy if not + available""" + _id = context.idmap.get((model._name, record['id'])) + if not _id: + _id = context.dummies.get((model._name, record['id'])) + if _id: + context.dummy_instances.append( + dummy_instance(*(context.field_context + (_id,))) + ) + if not _id: + mapping = self.import_field_mappings.filtered( + lambda x: x.model_id.model == model._name and + ( + not x.fields_id or + x.fields_id.name == context.field_context.field_name and + x.fields_id.model_id.model == + context.field_context.record_model + ) and + x.local_id and + (x.remote_id == record['id'] or not x.remote_id) + )[:1] + if mapping: + if mapping.local_id: + _id = mapping.local_id + context.idmap[(model._name, record['id'])] = _id + else: + _id = self._run_import_create_dummy( + context, model, record, forcecreate=True, + ) + if not _id: + xmlid = self.env['ir.model.data'].search([ + ('import_database_id', '=', self.id), + ('import_database_record_id', '=', record['id']), + ('model', '=', model._name), + ], limit=1) + if xmlid: + _id = xmlid.res_id + context.idmap[(model._name, record['id'])] = _id + if not _id and create_dummy: + _id = self._run_import_create_dummy(context, model, record) + return _id + + @api.multi + def _run_import_create_dummy( + self, context, model, record, forcecreate=False, + ): + """Either misuse some existing record or create an empty one to satisfy + required links""" + dummy = model.search([ + ( + 'id', 'not in', + [ + v for (model_name, remote_id), v + in context.dummies.iteritems() + if model_name == model._name + ] + ), + ], limit=1) + if dummy and not forcecreate: + context.dummies[mapping_key(model._name, record['id'])] = dummy.id + context.dummy_instances.append( + dummy_instance(*(context.field_context + (dummy.id,))) + ) + return dummy.id + required = [ + name + for name, field in model._fields.iteritems() + if field.required + ] + defaults = model.default_get(required) + values = {'id': record['id']} + for name, field in model._fields.iteritems(): + if name not in required or name in defaults: + continue + value = None + if field.type in ['char', 'text', 'html']: + value = '' + elif field.type in ['boolean']: + value = False + elif field.type in ['integer', 'float']: + value = 0 + elif model._fields[name].type in ['date', 'datetime']: + value = '2000-01-01' + elif field.type in ['many2one']: + new_context = context.with_field_context( + model._name, name, record['id'] + ) + value = self._run_import_get_record( + new_context, + self.env[model._fields[name].comodel_name], + {'id': record.get(name, [None])[0]}, + ) + elif field.type in ['selection'] and not callable(field.selection): + value = field.selection[0][0] + elif field.type in ['selection'] and callable(field.selection): + value = field.selection(model)[0][0] + # TODO: support more types, refactor to one function per type + values[name] = value + dummy = self._create_record(context, model, values) + context.dummies[mapping_key(model._name, record['id'])] = dummy.id + context.to_delete.setdefault(model._name, []) + context.to_delete[model._name].append(dummy.id) + context.dummy_instances.append( + dummy_instance(*(context.field_context + (dummy.id,))) + ) + return dummy.id + + @api.multi + def _run_import_map_values(self, context, data): + model = self.env[context.model_line.model_id.model] + for field_name in data.keys(): + if not isinstance( + model._fields[field_name], fields._Relational + ) or not data[field_name]: + continue + if model._fields[field_name].type == 'one2many': + # don't import one2many fields, use an own configuration + # for this + data.pop(field_name) + continue + ids = data[field_name] if ( + model._fields[field_name].type != 'many2one' + ) else [data[field_name][0]] + new_context = context.with_field_context( + model._name, field_name, data['id'] + ) + data[field_name] = [ + self._run_import_get_record( + new_context, + self.env[model._fields[field_name].comodel_name], + {'id': _id}, + create_dummy=model._fields[field_name].required, + ) + for _id in ids + ] + data[field_name] = filter(None, data[field_name]) + if model._fields[field_name].type == 'many2one': + if data[field_name]: + data[field_name] = data[field_name] and data[field_name][0] + else: + data[field_name] = None + else: + data[field_name] = [(6, 0, data[field_name])] + for mapping in self.import_field_mappings: + if mapping.model_id.model != model._name or not mapping.fields_id: + continue + if mapping.unique: + value = data.get(mapping.fields_id.name, '') + counter = 1 + while model.with_context(active_test=False).search([ + ( + mapping.fields_id.name, '=', + data.get(mapping.fields_id.name, value) + ), + ]): + data[mapping.fields_id.name] = '%s (%d)' % (value, counter) + counter += 1 + return data + + @api.multi + def _run_import_model_get_fields(self, context): + return { + name: field + for name, field + in self.env[context.model_line.model_id.model]._fields.iteritems() + if not field.compute or field.related + } + + @api.multi + def _run_import_model_cleanup_dummies( + self, context, model, remote_id, local_id + ): + for instance in context.dummy_instances: + if ( + instance.model_name != model._name or + instance.remote_id != remote_id + ): + continue + if not context.idmap.get(instance.remote_id): + continue + model = self.env[instance.model_name] + record = model.browse(context.idmap[instance.remote_id]) + field_name = instance.field_id.name + if record._fields[field_name].type == 'many2one': + record.write({field_name: local_id}) + elif record._fields[field_name].type == 'many2many': + record.write({field_name: [ + (3, context.idmap[remote_id]), + (4, local_id), + ]}) + else: + raise exceptions.UserError( + _('Unhandled field type %s') % + record._fields[field_name].type + ) + context.dummy_instances.remove(instance) + dummy_id = context.dummies[(record._model, remote_id)] + if dummy_id in context.to_delete: + model.browse(dummy_id).unlink() + del context.dummies[(record._model, remote_id)] + + def _get_connection(self): + self.ensure_one() + return Client(self.url, self.database, self.user, self.password) + + @api.constrains('url', 'database', 'user', 'password') + @api.multi + def _constrain_url(self): + for this in self: + if this == self.env.ref('base_import_odoo.demodb', False): + continue + if tools.config['test_enable']: + continue + if not this.password: + continue + this._get_connection() + + @api.depends('status_data') + @api.multi + def _compute_status_html(self): + for this in self: + if not this.status_data: + continue + this.status_html = self.env.ref( + 'base_import_odoo.view_import_odoo_database_qweb' + ).render({'object': this}) + + @api.depends('cronjob_id') + @api.multi + def _compute_cronjob_running(self): + for this in self: + if not this.cronjob_id: + continue + try: + with self.env.cr.savepoint(): + self.env.cr.execute( + 'select id from "%s" where id=%%s for update nowait' % + self.env['ir.cron']._table, + (this.cronjob_id.id,), log_exceptions=False, + ) + except psycopg2.OperationalError: + this.cronjob_running = True + + @api.multi + def _create_cronjob(self): + self.ensure_one() + return self.env['ir.cron'].create({ + 'name': self.display_name, + 'model': self._name, + 'function': '_run_import', + 'doall': True, + 'args': str((self.ids,)), + }) + + @api.multi + def name_get(self): + return [ + (this.id, '%s@%s, %s' % (this.user, this.url, this.database)) + for this in self + ] diff --git a/base_import_odoo/models/import_odoo_database_field.py b/base_import_odoo/models/import_odoo_database_field.py new file mode 100644 index 000000000..ff823a9b1 --- /dev/null +++ b/base_import_odoo/models/import_odoo_database_field.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from openerp import fields, models + + +class ImportOdooDatabaseField(models.Model): + _name = 'import.odoo.database.field' + _description = 'A field mapping for records in the remote database' + + database_id = fields.Many2one( + 'import.odoo.database', string='Database', required=True, + ondelete='cascade', + ) + model_id = fields.Many2one( + 'ir.model', string='Model', required=True, ondelete='cascade', + ) + fields_id = fields.Many2one( + 'ir.model.fields', string='Field', help='If set, the mapping is only ' + 'effective when setting said field', ondelete='cascade', + ) + unique = fields.Boolean( + 'Unique', help='If set on a char field, a number is appended until ' + 'the value is unique. Set this for fields with uniqueness constraints', + ) + # TODO: create a reference function field to set this conveniently + local_id = fields.Integer( + 'Local ID', help='If you leave this empty, a new record will be ' + 'created in the local database when this field is set on the remote ' + 'database' + ) + remote_id = fields.Integer( + 'Remote ID', help='If you leave this empty, every (set) field value ' + 'will be mapped to the local ID' + ) diff --git a/base_import_odoo/models/import_odoo_database_model.py b/base_import_odoo/models/import_odoo_database_model.py new file mode 100644 index 000000000..2394ebc65 --- /dev/null +++ b/base_import_odoo/models/import_odoo_database_model.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from openerp import fields, models + + +class ImportOdooDatabaseModel(models.Model): + _name = 'import.odoo.database.model' + _description = 'A model to import from a remote database' + _order = 'sequence' + + sequence = fields.Integer() + model_id = fields.Many2one( + 'ir.model', string='Model', required=True, ondelete='cascade', + ) + database_id = fields.Many2one( + 'import.odoo.database', string='Database', required=True, + ondelete='cascade', + ) + domain = fields.Char(help='Optional filter to import only a subset') diff --git a/base_import_odoo/models/ir_model_data.py b/base_import_odoo/models/ir_model_data.py new file mode 100644 index 000000000..2348232cd --- /dev/null +++ b/base_import_odoo/models/ir_model_data.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from openerp import fields, models + + +class IrModelData(models.Model): + _inherit = 'ir.model.data' + + import_database_id = fields.Many2one( + 'import.odoo.database', string='From remote database', + ) + import_database_record_id = fields.Integer('Remote database ID') diff --git a/base_import_odoo/security/ir.model.access.csv b/base_import_odoo/security/ir.model.access.csv new file mode 100644 index 000000000..b2413c5f8 --- /dev/null +++ b/base_import_odoo/security/ir.model.access.csv @@ -0,0 +1,4 @@ +"id","name","model_id:id","group_id:id","perm_read","perm_write","perm_create","perm_unlink" +access_import_odoo_database,access_import_odoo_database,model_import_odoo_database,base.group_system,1,1,1,1 +access_import_odoo_database_model,access_import_odoo_database_model,model_import_odoo_database_model,base.group_system,1,1,1,1 +access_import_odoo_database_field,access_import_odoo_database_field,model_import_odoo_database_field,base.group_system,1,1,1,1 diff --git a/base_import_odoo/static/description/icon.png b/base_import_odoo/static/description/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..3a0328b516c4980e8e44cdb63fd945757ddd132d GIT binary patch literal 9455 zcmW++2RxMjAAjx~&dlBk9S+%}OXg)AGE&Cb*&}d0jUxM@u(PQx^-s)697TX`ehR4?GS^qbkof1cslKgkU)h65qZ9Oc=ml_0temigYLJfnz{IDzUf>bGs4N!v3=Z3jMq&A#7%rM5eQ#dc?k~! zVpnB`o+K7|Al`Q_U;eD$B zfJtP*jH`siUq~{KE)`jP2|#TUEFGRryE2`i0**z#*^6~AI|YzIWy$Cu#CSLW3q=GA z6`?GZymC;dCPk~rBS%eCb`5OLr;RUZ;D`}um=H)BfVIq%7VhiMr)_#G0N#zrNH|__ zc+blN2UAB0=617@>_u;MPHN;P;N#YoE=)R#i$k_`UAA>WWCcEVMh~L_ zj--gtp&|K1#58Yz*AHCTMziU1Jzt_jG0I@qAOHsk$2}yTmVkBp_eHuY$A9)>P6o~I z%aQ?!(GqeQ-Y+b0I(m9pwgi(IIZZzsbMv+9w{PFtd_<_(LA~0H(xz{=FhLB@(1&qHA5EJw1>>=%q2f&^X>IQ{!GJ4e9U z&KlB)z(84HmNgm2hg2C0>WM{E(DdPr+EeU_N@57;PC2&DmGFW_9kP&%?X4}+xWi)( z;)z%wI5>D4a*5XwD)P--sPkoY(a~WBw;E~AW`Yue4kFa^LM3X`8x|}ZUeMnqr}>kH zG%WWW>3ml$Yez?i%)2pbKPI7?5o?hydokgQyZsNEr{a|mLdt;X2TX(#B1j35xPnPW z*bMSSOauW>o;*=kO8ojw91VX!qoOQb)zHJ!odWB}d+*K?#sY_jqPdg{Sm2HdYzdEx zOGVPhVRTGPtv0o}RfVP;Nd(|CB)I;*t&QO8h zFfekr30S!-LHmV_Su-W+rEwYXJ^;6&3|L$mMC8*bQptyOo9;>Qb9Q9`ySe3%V$A*9 zeKEe+b0{#KWGp$F+tga)0RtI)nhMa-K@JS}2krK~n8vJ=Ngm?R!9G<~RyuU0d?nz# z-5EK$o(!F?hmX*2Yt6+coY`6jGbb7tF#6nHA zuKk=GGJ;ZwON1iAfG$E#Y7MnZVmrY|j0eVI(DN_MNFJmyZ|;w4tf@=CCDZ#5N_0K= z$;R~bbk?}TpfDjfB&aiQ$VA}s?P}xPERJG{kxk5~R`iRS(SK5d+Xs9swCozZISbnS zk!)I0>t=A<-^z(cmSFz3=jZ23u13X><0b)P)^1T_))Kr`e!-pb#q&J*Q`p+B6la%C zuVl&0duN<;uOsB3%T9Fp8t{ED108<+W(nOZd?gDnfNBC3>M8WE61$So|P zVvqH0SNtDTcsUdzaMDpT=Ty0pDHHNL@Z0w$Y`XO z2M-_r1S+GaH%pz#Uy0*w$Vdl=X=rQXEzO}d6J^R6zjM1u&c9vYLvLp?W7w(?np9x1 zE_0JSAJCPB%i7p*Wvg)pn5T`8k3-uR?*NT|J`eS#_#54p>!p(mLDvmc-3o0mX*mp_ zN*AeS<>#^-{S%W<*mz^!X$w_2dHWpcJ6^j64qFBft-o}o_Vx80o0>}Du;>kLts;$8 zC`7q$QI(dKYG`Wa8#wl@V4jVWBRGQ@1dr-hstpQL)Tl+aqVpGpbSfN>5i&QMXfiZ> zaA?T1VGe?rpQ@;+pkrVdd{klI&jVS@I5_iz!=UMpTsa~mBga?1r}aRBm1WS;TT*s0f0lY=JBl66Upy)-k4J}lh=P^8(SXk~0xW=T9v*B|gzIhN z>qsO7dFd~mgxAy4V?&)=5ieYq?zi?ZEoj)&2o)RLy=@hbCRcfT5jigwtQGE{L*8<@Yd{zg;CsL5mvzfDY}P-wos_6PfprFVaeqNE%h zKZhLtcQld;ZD+>=nqN~>GvROfueSzJD&BE*}XfU|H&(FssBqY=hPCt`d zH?@s2>I(|;fcW&YM6#V#!kUIP8$Nkdh0A(bEVj``-AAyYgwY~jB zT|I7Bf@%;7aL7Wf4dZ%VqF$eiaC38OV6oy3Z#TER2G+fOCd9Iaoy6aLYbPTN{XRPz z;U!V|vBf%H!}52L2gH_+j;`bTcQRXB+y9onc^wLm5wi3-Be}U>k_u>2Eg$=k!(l@I zcCg+flakT2Nej3i0yn+g+}%NYb?ta;R?(g5SnwsQ49U8Wng8d|{B+lyRcEDvR3+`O{zfmrmvFrL6acVP%yG98X zo&+VBg@px@i)%o?dG(`T;n*$S5*rnyiR#=wW}}GsAcfyQpE|>a{=$Hjg=-*_K;UtD z#z-)AXwSRY?OPefw^iI+ z)AXz#PfEjlwTes|_{sB?4(O@fg0AJ^g8gP}ex9Ucf*@_^J(s_5jJV}c)s$`Myn|Kd z$6>}#q^n{4vN@+Os$m7KV+`}c%4)4pv@06af4-x5#wj!KKb%caK{A&Y#Rfs z-po?Dcb1({W=6FKIUirH&(yg=*6aLCekcKwyfK^JN5{wcA3nhO(o}SK#!CINhI`-I z1)6&n7O&ZmyFMuNwvEic#IiOAwNkR=u5it{B9n2sAJV5pNhar=j5`*N!Na;c7g!l$ z3aYBqUkqqTJ=Re-;)s!EOeij=7SQZ3Hq}ZRds%IM*PtM$wV z@;rlc*NRK7i3y5BETSKuumEN`Xu_8GP1Ri=OKQ$@I^ko8>H6)4rjiG5{VBM>B|%`&&s^)jS|-_95&yc=GqjNo{zFkw%%HHhS~e=s zD#sfS+-?*t|J!+ozP6KvtOl!R)@@-z24}`9{QaVLD^9VCSR2b`b!KC#o;Ki<+wXB6 zx3&O0LOWcg4&rv4QG0)4yb}7BFSEg~=IR5#ZRj8kg}dS7_V&^%#Do==#`u zpy6{ox?jWuR(;pg+f@mT>#HGWHAJRRDDDv~@(IDw&R>9643kK#HN`!1vBJHnC+RM&yIh8{gG2q zA%e*U3|N0XSRa~oX-3EAneep)@{h2vvd3Xvy$7og(sayr@95+e6~Xvi1tUqnIxoIH zVWo*OwYElb#uyW{Imam6f2rGbjR!Y3`#gPqkv57dB6K^wRGxc9B(t|aYDGS=m$&S!NmCtrMMaUg(c zc2qC=2Z`EEFMW-me5B)24AqF*bV5Dr-M5ig(l-WPS%CgaPzs6p_gnCIvTJ=Y<6!gT zVt@AfYCzjjsMEGi=rDQHo0yc;HqoRNnNFeWZgcm?f;cp(6CNylj36DoL(?TS7eU#+ z7&mfr#y))+CJOXQKUMZ7QIdS9@#-}7y2K1{8)cCt0~-X0O!O?Qx#E4Og+;A2SjalQ zs7r?qn0H044=sDN$SRG$arw~n=+T_DNdSrarmu)V6@|?1-ZB#hRn`uilTGPJ@fqEy zGt(f0B+^JDP&f=r{#Y_wi#AVDf-y!RIXU^0jXsFpf>=Ji*TeqSY!H~AMbJdCGLhC) zn7Rx+sXw6uYj;WRYrLd^5IZq@6JI1C^YkgnedZEYy<&4(z%Q$5yv#Boo{AH8n$a zhb4Y3PWdr269&?V%uI$xMcUrMzl=;w<_nm*qr=c3Rl@i5wWB;e-`t7D&c-mcQl7x! zZWB`UGcw=Y2=}~wzrfLx=uet<;m3~=8I~ZRuzvMQUQdr+yTV|ATf1Uuomr__nDf=X zZ3WYJtHp_ri(}SQAPjv+Y+0=fH4krOP@S&=zZ-t1jW1o@}z;xk8 z(Nz1co&El^HK^NrhVHa-_;&88vTU>_J33=%{if;BEY*J#1n59=07jrGQ#IP>@u#3A z;!q+E1Rj3ZJ+!4bq9F8PXJ@yMgZL;>&gYA0%_Kbi8?S=XGM~dnQZQ!yBSgcZhY96H zrWnU;k)qy`rX&&xlDyA%(a1Hhi5CWkmg(`Gb%m(HKi-7Z!LKGRP_B8@`7&hdDy5n= z`OIxqxiVfX@OX1p(mQu>0Ai*v_cTMiw4qRt3~NBvr9oBy0)r>w3p~V0SCm=An6@3n)>@z!|o-$HvDK z|3D2ZMJkLE5loMKl6R^ez@Zz%S$&mbeoqH5`Bb){Ei21q&VP)hWS2tjShfFtGE+$z zzCR$P#uktu+#!w)cX!lWN1XU%K-r=s{|j?)Akf@q#3b#{6cZCuJ~gCxuMXRmI$nGtnH+-h z+GEi!*X=AP<|fG`1>MBdTb?28JYc=fGvAi2I<$B(rs$;eoJCyR6_bc~p!XR@O-+sD z=eH`-ye})I5ic1eL~TDmtfJ|8`0VJ*Yr=hNCd)G1p2MMz4C3^Mj?7;!w|Ly%JqmuW zlIEW^Ft%z?*|fpXda>Jr^1noFZEwFgVV%|*XhH@acv8rdGxeEX{M$(vG{Zw+x(ei@ zmfXb22}8-?Fi`vo-YVrTH*C?a8%M=Hv9MqVH7H^J$KsD?>!SFZ;ZsvnHr_gn=7acz z#W?0eCdVhVMWN12VV^$>WlQ?f;P^{(&pYTops|btm6aj>_Uz+hqpGwB)vWp0Cf5y< zft8-je~nn?W11plq}N)4A{l8I7$!ks_x$PXW-2XaRFswX_BnF{R#6YIwMhAgd5F9X zGmwdadS6(a^fjHtXg8=l?Rc0Sm%hk6E9!5cLVloEy4eh(=FwgP`)~I^5~pBEWo+F6 zSf2ncyMurJN91#cJTy_u8Y}@%!bq1RkGC~-bV@SXRd4F{R-*V`bS+6;W5vZ(&+I<9$;-V|eNfLa5n-6% z2(}&uGRF;p92eS*sE*oR$@pexaqr*meB)VhmIg@h{uzkk$9~qh#cHhw#>O%)b@+(| z^IQgqzuj~Sk(J;swEM-3TrJAPCq9k^^^`q{IItKBRXYe}e0Tdr=Huf7da3$l4PdpwWDop%^}n;dD#K4s#DYA8SHZ z&1!riV4W4R7R#C))JH1~axJ)RYnM$$lIR%6fIVA@zV{XVyx}C+a-Dt8Y9M)^KU0+H zR4IUb2CJ{Hg>CuaXtD50jB(_Tcx=Z$^WYu2u5kubqmwp%drJ6 z?Fo40g!Qd<-l=TQxqHEOuPX0;^z7iX?Ke^a%XT<13TA^5`4Xcw6D@Ur&VT&CUe0d} z1GjOVF1^L@>O)l@?bD~$wzgf(nxX1OGD8fEV?TdJcZc2KoUe|oP1#=$$7ee|xbY)A zDZq+cuTpc(fFdj^=!;{k03C69lMQ(|>uhRfRu%+!k&YOi-3|1QKB z z?n?eq1XP>p-IM$Z^C;2L3itnbJZAip*Zo0aw2bs8@(s^~*8T9go!%dHcAz2lM;`yp zD=7&xjFV$S&5uDaiScyD?B-i1ze`+CoRtz`Wn+Zl&#s4&}MO{@N!ufrzjG$B79)Y2d3tBk&)TxUTw@QS0TEL_?njX|@vq?Uz(nBFK5Pq7*xj#u*R&i|?7+6# z+|r_n#SW&LXhtheZdah{ZVoqwyT{D>MC3nkFF#N)xLi{p7J1jXlmVeb;cP5?e(=f# zuT7fvjSbjS781v?7{)-X3*?>tq?)Yd)~|1{BDS(pqC zC}~H#WXlkUW*H5CDOo<)#x7%RY)A;ShGhI5s*#cRDA8YgqG(HeKDx+#(ZQ?386dv! zlXCO)w91~Vw4AmOcATuV653fa9R$fyK8ul%rG z-wfS zihugoZyr38Im?Zuh6@RcF~t1anQu7>#lPpb#}4cOA!EM11`%f*07RqOVkmX{p~KJ9 z^zP;K#|)$`^Rb{rnHGH{~>1(fawV0*Z#)}M`m8-?ZJV<+e}s9wE# z)l&az?w^5{)`S(%MRzxdNqrs1n*-=jS^_jqE*5XDrA0+VE`5^*p3CuM<&dZEeCjoz zR;uu_H9ZPZV|fQq`Cyw4nscrVwi!fE6ciMmX$!_hN7uF;jjKG)d2@aC4ropY)8etW=xJvni)8eHi`H$%#zn^WJ5NLc-rqk|u&&4Z6fD_m&JfSI1Bvb?b<*n&sfl0^t z=HnmRl`XrFvMKB%9}>PaA`m-fK6a0(8=qPkWS5bb4=v?XcWi&hRY?O5HdulRi4?fN zlsJ*N-0Qw+Yic@s0(2uy%F@ib;GjXt01Fmx5XbRo6+n|pP(&nodMoap^z{~q ziEeaUT@Mxe3vJSfI6?uLND(CNr=#^W<1b}jzW58bIfyWTDle$mmS(|x-0|2UlX+9k zQ^EX7Nw}?EzVoBfT(-LT|=9N@^hcn-_p&sqG z&*oVs2JSU+N4ZD`FhCAWaS;>|wH2G*Id|?pa#@>tyxX`+4HyIArWDvVrX)2WAOQff z0qyHu&-S@i^MS-+j--!pr4fPBj~_8({~e1bfcl0wI1kaoN>mJL6KUPQm5N7lB(ui1 zE-o%kq)&djzWJ}ob<-GfDlkB;F31j-VHKvQUGQ3sp`CwyGJk_i!y^sD0fqC@$9|jO zOqN!r!8-p==F@ZVP=U$qSpY(gQ0)59P1&t@y?5rvg<}E+GB}26NYPp4f2YFQrQtot5mn3wu_qprZ=>Ig-$ zbW26Ws~IgY>}^5w`vTB(G`PTZaDiGBo5o(tp)qli|NeV( z@H_=R8V39rt5J5YB2Ky?4eJJ#b`_iBe2ot~6%7mLt5t8Vwi^Jy7|jWXqa3amOIoRb zOr}WVFP--DsS`1WpN%~)t3R!arKF^Q$e12KEqU36AWwnCBICpH4XCsfnyrHr>$I$4 z!DpKX$OKLWarN7nv@!uIA+~RNO)l$$w}p(;b>mx8pwYvu;dD_unryX_NhT8*Tj>BTrTTL&!?O+%Rv;b?B??gSzdp?6Uug9{ zd@V08Z$BdI?fpoCS$)t4mg4rT8Q_I}h`0d-vYZ^|dOB*Q^S|xqTV*vIg?@fVFSmMpaw0qtTRbx} z({Pg?#{2`sc9)M5N$*N|4;^t$+QP?#mov zGVC@I*lBVrOU-%2y!7%)fAKjpEFsgQc4{amtiHb95KQEwvf<(3T<9-Zm$xIew#P22 zc2Ix|App^>v6(3L_MCU0d3W##AB0M~3D00EWoKZqsJYT(#@w$Y_H7G22M~ApVFTRHMI_3be)Lkn#0F*V8Pq zc}`Cjy$bE;FJ6H7p=0y#R>`}-m4(0F>%@P|?7fx{=R^uFdISRnZ2W_xQhD{YuR3t< z{6yxu=4~JkeA;|(J6_nv#>Nvs&FuLA&PW^he@t(UwFFE8)|a!R{`E`K`i^ZnyE4$k z;(749Ix|oi$c3QbEJ3b~D_kQsPz~fIUKym($a_7dJ?o+40*OLl^{=&oq$<#Q(yyrp z{J-FAniyAw9tPbe&IhQ|a`DqFTVQGQ&Gq3!C2==4x{6EJwiPZ8zub-iXoUtkJiG{} zPaR&}_fn8_z~(=;5lD-aPWD3z8PZS@AaUiomF!G8I}Mf>e~0g#BelA-5#`cj;O5>N Xviia!U7SGha1wx#SCgwmn*{w2TRX*I literal 0 HcmV?d00001 diff --git a/base_import_odoo/tests/__init__.py b/base_import_odoo/tests/__init__.py new file mode 100644 index 000000000..905385a18 --- /dev/null +++ b/base_import_odoo/tests/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from . import test_base_import_odoo diff --git a/base_import_odoo/tests/test_base_import_odoo.py b/base_import_odoo/tests/test_base_import_odoo.py new file mode 100644 index 000000000..f3201ecdc --- /dev/null +++ b/base_import_odoo/tests/test_base_import_odoo.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from mock import patch +from openerp.tests.common import TransactionCase, post_install, at_install + + +class TestBaseImportOdoo(TransactionCase): + @at_install(False) + @post_install(True) + @patch('erppeek.Client.__init__', side_effect=lambda *args: None) + def test_base_import_odoo(self, mock_client_init): + # the mocked functions simply search/read in the current database + # the effect then should be that the models in question are duplicated, + # we just need to try not to be confused by the fact that local and + # remote ids are the same + def _mock_search( + model, domain, offset=0, limit=None, order=None, context=None, + ): + return self.env[model].with_context( + **(context or self.env.context) + ).search( + domain, offset=offset, limit=limit, order=order, + ).ids + + def _mock_read( + model, domain_or_ids, fields=None, offset=0, limit=None, + order=None, context=None, + ): + return self.env[model].with_context( + **(context or self.env.context) + ).browse(domain_or_ids).read(fields=fields) + + for dummy in range(2): + # we run this two times to enter the code path where xmlids exist + self.env.ref('base_import_odoo.demodb').write({ + 'password': 'admin', + }) + with patch('erppeek.Client.search', side_effect=_mock_search): + with patch('erppeek.Client.read', side_effect=_mock_read): + self.env.ref('base_import_odoo.demodb')._run_import() + # here the actual test begins - check that we created new + # objects, check xmlids, check values, check if dummies are + # cleaned up/replaced + self.assertNotEqual( + self.env.ref(self._get_xmlid('base.user_demo')), + self.env.ref('base.user_demo'), + ) + self.assertEqual( + dict(self.env.ref(self._get_xmlid('base.user_demo'))._cache), + dict(self.env.ref('base.user_demo')._cache), + ) + # TODO: test much more + demodb = self.env.ref('base_import_odoo.demodb') + demodb.action_import() + self.assertTrue(demodb.cronjob_id) + demodb.cronjob_id.write({'active': False}) + demodb.action_import() + self.assertTrue(demodb.cronjob_id.active) + self.assertFalse(demodb.cronjob_running) + + def _get_xmlid(self, remote_xmlid): + remote_obj = self.env.ref(remote_xmlid) + return 'base_import_odoo.%d-%s-%s' % ( + self.env.ref('base_import_odoo.demodb').id, + remote_obj._name.replace('.', '_'), + remote_obj.id, + ) diff --git a/base_import_odoo/views/import_odoo_database.xml b/base_import_odoo/views/import_odoo_database.xml new file mode 100644 index 000000000..d5c67077b --- /dev/null +++ b/base_import_odoo/views/import_odoo_database.xml @@ -0,0 +1,89 @@ + + + + + import.odoo.database + + + + + + + + + import.odoo.database + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + diff --git a/base_import_odoo/views/menu.xml b/base_import_odoo/views/menu.xml new file mode 100644 index 000000000..42b79797a --- /dev/null +++ b/base_import_odoo/views/menu.xml @@ -0,0 +1,12 @@ + + + + + + + + From 2c1b13b34438d09ebb30c9fa2d218a7663d47095 Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Wed, 11 Oct 2017 18:38:32 +0200 Subject: [PATCH 02/21] [ADD] allow mapping records by equal field values --- base_import_odoo/__openerp__.py | 1 + .../demo/import_odoo_database_field.xml | 42 +++++++- .../demo/import_odoo_database_model.xml | 6 ++ .../models/import_odoo_database.py | 95 +++++++++++++------ .../models/import_odoo_database_field.py | 17 +++- .../tests/test_base_import_odoo.py | 36 ++++++- .../views/import_odoo_database.xml | 10 +- .../views/import_odoo_database_field.xml | 42 ++++++++ 8 files changed, 199 insertions(+), 50 deletions(-) create mode 100644 base_import_odoo/views/import_odoo_database_field.xml diff --git a/base_import_odoo/__openerp__.py b/base_import_odoo/__openerp__.py index baaba59e5..cfc169e3c 100644 --- a/base_import_odoo/__openerp__.py +++ b/base_import_odoo/__openerp__.py @@ -18,6 +18,7 @@ "demo/import_odoo_database_model.xml", ], "data": [ + "views/import_odoo_database_field.xml", "security/ir.model.access.csv", "views/import_odoo_database.xml", "views/menu.xml", diff --git a/base_import_odoo/demo/import_odoo_database_field.xml b/base_import_odoo/demo/import_odoo_database_field.xml index e47c6d070..af9e3d137 100644 --- a/base_import_odoo/demo/import_odoo_database_field.xml +++ b/base_import_odoo/demo/import_odoo_database_field.xml @@ -1,23 +1,59 @@ - + + fixed + + + + + + + fixed + + + + + + + fixed + + + + + + + fixed + + + fixed + + + + + fixed + unique - - + + + + + by_field + + diff --git a/base_import_odoo/demo/import_odoo_database_model.xml b/base_import_odoo/demo/import_odoo_database_model.xml index c9c856766..a10fb3bee 100644 --- a/base_import_odoo/demo/import_odoo_database_model.xml +++ b/base_import_odoo/demo/import_odoo_database_model.xml @@ -13,5 +13,11 @@ [(1, '=', 1)] + + 3 + + + [(1, '=', 1)] + diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py index 8a15b73f4..8d462bd24 100644 --- a/base_import_odoo/models/import_odoo_database.py +++ b/base_import_odoo/models/import_odoo_database.py @@ -220,25 +220,9 @@ class ImportOdooDatabase(models.Model): dummy_instance(*(context.field_context + (_id,))) ) if not _id: - mapping = self.import_field_mappings.filtered( - lambda x: x.model_id.model == model._name and - ( - not x.fields_id or - x.fields_id.name == context.field_context.field_name and - x.fields_id.model_id.model == - context.field_context.record_model - ) and - x.local_id and - (x.remote_id == record['id'] or not x.remote_id) - )[:1] - if mapping: - if mapping.local_id: - _id = mapping.local_id - context.idmap[(model._name, record['id'])] = _id - else: - _id = self._run_import_create_dummy( - context, model, record, forcecreate=True, - ) + _id = self._run_import_get_record_mapping( + context, model, record, create_dummy=create_dummy, + ) if not _id: xmlid = self.env['ir.model.data'].search([ ('import_database_id', '=', self.id), @@ -252,6 +236,48 @@ class ImportOdooDatabase(models.Model): _id = self._run_import_create_dummy(context, model, record) return _id + @api.multi + def _run_import_get_record_mapping( + self, context, model, record, create_dummy=True, + ): + current_field = self.env['ir.model.fields'].search([ + ('name', '=', context.field_context.field_name), + ('model_id.model', '=', context.field_context.record_model), + ]) + mappings = self.import_field_mappings.filtered( + lambda x: + x.mapping_type == 'fixed' and + x.model_id.model == model._name and + ( + not x.field_ids or current_field in x.field_ids + ) and x.local_id and + (x.remote_id == record['id'] or not x.remote_id) or + x.mapping_type == 'by_field' and + x.model_id.model == model._name + ) + _id = None + for mapping in mappings: + if mapping.mapping_type == 'fixed': + assert mapping.local_id + _id = mapping.local_id + context.idmap[(model._name, record['id'])] = _id + break + elif mapping.mapping_type == 'by_field': + assert mapping.field_ids + if len(record) == 1: + continue + records = model.search([ + (field.name, '=', record[field.name]) + for field in mapping.field_ids + ], limit=1) + if records: + _id = records.id + context.idmap[(model._name, record['id'])] = _id + break + else: + raise exceptions.UserError(_('Unknown mapping')) + return _id + @api.multi def _run_import_create_dummy( self, context, model, record, forcecreate=False, @@ -265,6 +291,12 @@ class ImportOdooDatabase(models.Model): v for (model_name, remote_id), v in context.dummies.iteritems() if model_name == model._name + ] + + [ + mapping.local_id for mapping + in self.import_field_mappings + if mapping.model_id.model == model._name and + mapping.local_id ] ), ], limit=1) @@ -336,12 +368,15 @@ class ImportOdooDatabase(models.Model): new_context = context.with_field_context( model._name, field_name, data['id'] ) + comodel = self.env[model._fields[field_name].comodel_name] data[field_name] = [ self._run_import_get_record( - new_context, - self.env[model._fields[field_name].comodel_name], - {'id': _id}, - create_dummy=model._fields[field_name].required, + new_context, comodel, {'id': _id}, + create_dummy=model._fields[field_name].required or + any( + m.model_id._name == comodel._name + for m in self.import_line_ids + ), ) for _id in ids ] @@ -354,18 +389,16 @@ class ImportOdooDatabase(models.Model): else: data[field_name] = [(6, 0, data[field_name])] for mapping in self.import_field_mappings: - if mapping.model_id.model != model._name or not mapping.fields_id: + if mapping.model_id.model != model._name or\ + mapping.mapping_type != 'unique': continue - if mapping.unique: - value = data.get(mapping.fields_id.name, '') + for field in mapping.field_ids: + value = data.get(field.name, '') counter = 1 while model.with_context(active_test=False).search([ - ( - mapping.fields_id.name, '=', - data.get(mapping.fields_id.name, value) - ), + (field.name, '=', data.get(field.name, value)), ]): - data[mapping.fields_id.name] = '%s (%d)' % (value, counter) + data[field.name] = '%s (%d)' % (value, counter) counter += 1 return data diff --git a/base_import_odoo/models/import_odoo_database_field.py b/base_import_odoo/models/import_odoo_database_field.py index ff823a9b1..95d9b1959 100644 --- a/base_import_odoo/models/import_odoo_database_field.py +++ b/base_import_odoo/models/import_odoo_database_field.py @@ -7,7 +7,9 @@ from openerp import fields, models class ImportOdooDatabaseField(models.Model): _name = 'import.odoo.database.field' _description = 'A field mapping for records in the remote database' + _order = 'database_id, sequence' + sequence = fields.Integer() database_id = fields.Many2one( 'import.odoo.database', string='Database', required=True, ondelete='cascade', @@ -15,14 +17,11 @@ class ImportOdooDatabaseField(models.Model): model_id = fields.Many2one( 'ir.model', string='Model', required=True, ondelete='cascade', ) - fields_id = fields.Many2one( + model = fields.Char(related=['model_id', 'model']) + field_ids = fields.Many2many( 'ir.model.fields', string='Field', help='If set, the mapping is only ' 'effective when setting said field', ondelete='cascade', ) - unique = fields.Boolean( - 'Unique', help='If set on a char field, a number is appended until ' - 'the value is unique. Set this for fields with uniqueness constraints', - ) # TODO: create a reference function field to set this conveniently local_id = fields.Integer( 'Local ID', help='If you leave this empty, a new record will be ' @@ -33,3 +32,11 @@ class ImportOdooDatabaseField(models.Model): 'Remote ID', help='If you leave this empty, every (set) field value ' 'will be mapped to the local ID' ) + mapping_type = fields.Selection( + [ + ('fixed', 'Fixed'), + ('by_field', 'Based on equal fields'), + ('unique', 'Unique'), + ], + string='Type', required=True, default='fixed', + ) diff --git a/base_import_odoo/tests/test_base_import_odoo.py b/base_import_odoo/tests/test_base_import_odoo.py index f3201ecdc..3a9f3a186 100644 --- a/base_import_odoo/tests/test_base_import_odoo.py +++ b/base_import_odoo/tests/test_base_import_odoo.py @@ -31,6 +31,9 @@ class TestBaseImportOdoo(TransactionCase): **(context or self.env.context) ).browse(domain_or_ids).read(fields=fields) + group_count = self.env['res.groups'].search([], count=True) + user_count = self.env['res.users'].search([], count=True) + run = 1 for dummy in range(2): # we run this two times to enter the code path where xmlids exist self.env.ref('base_import_odoo.demodb').write({ @@ -46,11 +49,32 @@ class TestBaseImportOdoo(TransactionCase): self.env.ref(self._get_xmlid('base.user_demo')), self.env.ref('base.user_demo'), ) + # check that the imported scalars are equal + fields = ['name', 'email', 'signature', 'active'] + ( + self.env.ref(self._get_xmlid('base.user_demo')) + + self.env.ref('base.user_demo') + ).read(fields) self.assertEqual( - dict(self.env.ref(self._get_xmlid('base.user_demo'))._cache), - dict(self.env.ref('base.user_demo')._cache), + self._get_cache(self._get_xmlid('base.user_demo'), fields), + self._get_cache('base.user_demo', fields), + ) + # check that links are correctly mapped + self.assertEqual( + self.env.ref(self._get_xmlid('base.user_demo')).partner_id, + self.env.ref(self._get_xmlid('base.partner_demo')) + ) + # no new groups because they should be mapped by name + self.assertEqual( + group_count, self.env['res.groups'].search([], count=True) + ) + # all users save for root should be duplicated for every run + self.assertEqual( + self.env['res.users'].search([], count=True), + user_count + (user_count - 1) * run, ) # TODO: test much more + run += 1 demodb = self.env.ref('base_import_odoo.demodb') demodb.action_import() self.assertTrue(demodb.cronjob_id) @@ -66,3 +90,11 @@ class TestBaseImportOdoo(TransactionCase): remote_obj._name.replace('.', '_'), remote_obj.id, ) + + def _get_cache(self, xmlid, fields): + record = self.env.ref(xmlid) + return { + field_name: record._cache[field_name] + for field_name in record._fields + if field_name in fields + } diff --git a/base_import_odoo/views/import_odoo_database.xml b/base_import_odoo/views/import_odoo_database.xml index d5c67077b..8e293a722 100644 --- a/base_import_odoo/views/import_odoo_database.xml +++ b/base_import_odoo/views/import_odoo_database.xml @@ -35,15 +35,7 @@ - - - - - - - - - + diff --git a/base_import_odoo/views/import_odoo_database_field.xml b/base_import_odoo/views/import_odoo_database_field.xml new file mode 100644 index 000000000..061b63e50 --- /dev/null +++ b/base_import_odoo/views/import_odoo_database_field.xml @@ -0,0 +1,42 @@ + + + + + import.odoo.database.field + +
+ + + + + +
+ When a record of this model is imported, it will be replaced with the record you select as local ID. If you select a field and/or a remote ID, this replacement is only effective when setting the specified field and/or when the remote value is the specified record. +
+
+ Select fields which must be equal to treat a pair of remote and local records of this model as being equal. +
+
+ Select fields for which to generate unique values during import. You'll need this for res.users#login for example. +
+ + + + + +
+ + + + import.odoo.database.field + + + + + + + + + + + From 3f8ec60e00593853fcf7dd7bbe60bb6e1726e21c Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Wed, 11 Oct 2017 19:08:41 +0200 Subject: [PATCH 03/21] [FIX] support firefox --- base_import_odoo/views/import_odoo_database.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_import_odoo/views/import_odoo_database.xml b/base_import_odoo/views/import_odoo_database.xml index 8e293a722..e31548d93 100644 --- a/base_import_odoo/views/import_odoo_database.xml +++ b/base_import_odoo/views/import_odoo_database.xml @@ -69,7 +69,7 @@
- + / done
From dc6d022255026a74c80e0cd8c3251e4739ecade8 Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Wed, 11 Oct 2017 19:38:17 +0200 Subject: [PATCH 04/21] [IMP] community review --- base_import_odoo/README.rst | 1 + .../models/import_odoo_database.py | 21 +++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/base_import_odoo/README.rst b/base_import_odoo/README.rst index 73c0fee64..fc58abb1c 100644 --- a/base_import_odoo/README.rst +++ b/base_import_odoo/README.rst @@ -53,6 +53,7 @@ Known issues / Roadmap * Yes of course this duplicates a lot of connector functionality. Rewrite this with the connector framework, probably collaborate with https://github.com/OCA/connector-odoo2odoo * Do something with workflows +* Support reference fields, while being at it refactor _run_import_map_values to call a function per field type * Probably it's safer and faster to disable recomputation during import, and recompute all fields afterwards Bug Tracker diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py index 8d462bd24..50b87b982 100644 --- a/base_import_odoo/models/import_odoo_database.py +++ b/base_import_odoo/models/import_odoo_database.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- # © 2017 Therp BV # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import logging try: from erppeek import Client except: - pass + logging.debug('Unable to import erppeek') import psycopg2 import traceback from openerp import _, api, exceptions, fields, models, tools @@ -123,12 +124,15 @@ class ImportOdooDatabase(models.Model): for model_line in self.import_line_ids: model = self.env[model_line.model_id.model] done[model._name] = 0 + chunk_len = commit and (commit_threshold or 1) or len( + remote_ids[model._name] + ) for start_index in range( - len(remote_ids[model._name]) / commit_threshold + 1 + len(remote_ids[model._name]) / chunk_len + 1 ): - index = start_index * commit_threshold - ids = remote_ids[model._name][index:index + commit_threshold] + index = start_index * chunk_len + ids = remote_ids[model._name][index:index + chunk_len] context = ImportContext( remote, model_line, ids, idmap, dummies, dummy_instances, to_delete, field_context(None, None, None), @@ -289,7 +293,7 @@ class ImportOdooDatabase(models.Model): 'id', 'not in', [ v for (model_name, remote_id), v - in context.dummies.iteritems() + in context.dummies.items() if model_name == model._name ] + [ @@ -308,12 +312,12 @@ class ImportOdooDatabase(models.Model): return dummy.id required = [ name - for name, field in model._fields.iteritems() + for name, field in model._fields.items() if field.required ] defaults = model.default_get(required) values = {'id': record['id']} - for name, field in model._fields.iteritems(): + for name, field in model._fields.items(): if name not in required or name in defaults: continue value = None @@ -338,7 +342,6 @@ class ImportOdooDatabase(models.Model): value = field.selection[0][0] elif field.type in ['selection'] and callable(field.selection): value = field.selection(model)[0][0] - # TODO: support more types, refactor to one function per type values[name] = value dummy = self._create_record(context, model, values) context.dummies[mapping_key(model._name, record['id'])] = dummy.id @@ -407,7 +410,7 @@ class ImportOdooDatabase(models.Model): return { name: field for name, field - in self.env[context.model_line.model_id.model]._fields.iteritems() + in self.env[context.model_line.model_id.model]._fields.items() if not field.compute or field.related } From 60cba35fbdc50e8c498d91d15d4a61d5d3741702 Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Thu, 12 Oct 2017 09:07:22 +0200 Subject: [PATCH 05/21] [RFR] use odoorpc --- .travis.yml | 1 - base_import_odoo/__openerp__.py | 2 +- .../models/import_odoo_database.py | 26 ++++++++++---- .../tests/test_base_import_odoo.py | 34 +++++++------------ requirements.txt | 1 + 5 files changed, 34 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index 50f6081b7..d3e355a6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,6 @@ install: - export PATH=${HOME}/maintainer-quality-tools/travis:${PATH} - travis_install_nightly - printf '[options]\n\nrunning_env = dev\n' > ${HOME}/.openerp_serverrc - - pip install erppeek - ln -s ${TRAVIS_BUILD_DIR}/server_environment_files_sample ${TRAVIS_BUILD_DIR}/server_environment_files script: - travis_run_tests diff --git a/base_import_odoo/__openerp__.py b/base_import_odoo/__openerp__.py index cfc169e3c..5c40fc3a8 100644 --- a/base_import_odoo/__openerp__.py +++ b/base_import_odoo/__openerp__.py @@ -25,6 +25,6 @@ ], "installable": True, "external_dependencies": { - "python": ['erppeek'], + "python": ['odoorpc'], }, } diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py index 50b87b982..2b99d2133 100644 --- a/base_import_odoo/models/import_odoo_database.py +++ b/base_import_odoo/models/import_odoo_database.py @@ -3,11 +3,12 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). import logging try: - from erppeek import Client + import odoorpc except: - logging.debug('Unable to import erppeek') + logging.error('Unable to import odoorpc') import psycopg2 import traceback +from urlparse import urlparse from openerp import _, api, exceptions, fields, models, tools from collections import namedtuple @@ -105,8 +106,8 @@ class ImportOdooDatabase(models.Model): self.env.cr.commit() for model_line in self.import_line_ids: model = model_line.model_id - remote_ids[model.model] = remote.search( - model.model, + remote_ids[model.model] = remote.execute( + model.model, 'search', tools.safe_eval(model_line.domain) if model_line.domain else [] ) remote_counts[model.model] = len(remote_ids[model.model]) @@ -159,8 +160,8 @@ class ImportOdooDatabase(models.Model): """Import records of a configured model""" model = self.env[context.model_line.model_id.model] fields = self._run_import_model_get_fields(context) - for data in context.remote.read( - model._name, context.ids, fields.keys() + for data in context.remote.execute( + model._name, 'read', context.ids, fields.keys() ): self._run_import_get_record(context, model, data) if (model._name, data['id']) in context.idmap: @@ -449,7 +450,18 @@ class ImportOdooDatabase(models.Model): def _get_connection(self): self.ensure_one() - return Client(self.url, self.database, self.user, self.password) + url = urlparse(self.url) + hostport = url.netloc.split(':') + if len(hostport) == 1: + hostport.append('80') + host, port = hostport + remote = odoorpc.ODOO( + host, + protocol='jsonrpc+ssl' if url.scheme == 'https' else 'jsonrpc', + port=int(port), + ) + remote.login(self.database, self.user, self.password) + return remote @api.constrains('url', 'database', 'user', 'password') @api.multi diff --git a/base_import_odoo/tests/test_base_import_odoo.py b/base_import_odoo/tests/test_base_import_odoo.py index 3a9f3a186..76739fe56 100644 --- a/base_import_odoo/tests/test_base_import_odoo.py +++ b/base_import_odoo/tests/test_base_import_odoo.py @@ -8,28 +8,21 @@ from openerp.tests.common import TransactionCase, post_install, at_install class TestBaseImportOdoo(TransactionCase): @at_install(False) @post_install(True) - @patch('erppeek.Client.__init__', side_effect=lambda *args: None) - def test_base_import_odoo(self, mock_client_init): + @patch( + 'odoorpc.ODOO.__init__', + side_effect=lambda self, *args, **kwargs: None, + ) + @patch('odoorpc.ODOO.login', side_effect=lambda *args: None) + def test_base_import_odoo(self, mock_client, mock_client_login): # the mocked functions simply search/read in the current database # the effect then should be that the models in question are duplicated, # we just need to try not to be confused by the fact that local and # remote ids are the same - def _mock_search( - model, domain, offset=0, limit=None, order=None, context=None, - ): - return self.env[model].with_context( - **(context or self.env.context) - ).search( - domain, offset=offset, limit=limit, order=order, - ).ids - - def _mock_read( - model, domain_or_ids, fields=None, offset=0, limit=None, - order=None, context=None, - ): - return self.env[model].with_context( - **(context or self.env.context) - ).browse(domain_or_ids).read(fields=fields) + def _mock_execute(model, method, *args): + if method == 'read': + return self.env[model].browse(args[0]).read(fields=args[1]) + if method == 'search': + return self.env[model].search(args[0]).ids group_count = self.env['res.groups'].search([], count=True) user_count = self.env['res.users'].search([], count=True) @@ -39,9 +32,8 @@ class TestBaseImportOdoo(TransactionCase): self.env.ref('base_import_odoo.demodb').write({ 'password': 'admin', }) - with patch('erppeek.Client.search', side_effect=_mock_search): - with patch('erppeek.Client.read', side_effect=_mock_read): - self.env.ref('base_import_odoo.demodb')._run_import() + with patch('odoorpc.ODOO.execute', side_effect=_mock_execute): + self.env.ref('base_import_odoo.demodb')._run_import() # here the actual test begins - check that we created new # objects, check xmlids, check values, check if dummies are # cleaned up/replaced diff --git a/requirements.txt b/requirements.txt index 2b444a19a..08b8b0254 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ acme_tiny IPy python-json-logger + odoorpc From 753d5e25c72d6d1c08400c192d8fe192ba4eabcc Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Thu, 12 Oct 2017 10:07:53 +0200 Subject: [PATCH 06/21] [ADD] make it possible to import res_model,res_id references --- base_import_odoo/README.rst | 4 +- base_import_odoo/__openerp__.py | 1 + .../demo/import_odoo_database_field.xml | 7 ++++ .../demo/import_odoo_database_model.xml | 6 +++ base_import_odoo/demo/ir_attachment.xml | 11 +++++ .../models/import_odoo_database.py | 41 ++++++++++++++----- .../models/import_odoo_database_field.py | 28 ++++++++++++- .../tests/test_base_import_odoo.py | 25 ++++++----- .../views/import_odoo_database_field.xml | 7 +++- 9 files changed, 107 insertions(+), 23 deletions(-) create mode 100644 base_import_odoo/demo/ir_attachment.xml diff --git a/base_import_odoo/README.rst b/base_import_odoo/README.rst index fc58abb1c..3ff3d68b6 100644 --- a/base_import_odoo/README.rst +++ b/base_import_odoo/README.rst @@ -31,7 +31,9 @@ The module doesn't import one2many fields, if you want to have those, add the mo If you don't fill in a remote ID, the addon will use the configured local ID for every record of the model (this way, you can for example map all users in the remote system to some import user in the current system). -For fields that have a uniqueness constraint (like `res.users#login`), set the flag `unique`, then the import will generate a unique value for this field. +For fields that have a uniqueness constraint (like `res.users#login`), create a field mapping if type `unique`, then the import will generate a unique value for this field. + +For models using references with two fields (like `ir.attachment`), create a field mapping of type `by reference` and select the two fields involved. The import will transform known ids (=ids of models you import) to the respective local id, and clean out the model/id fields for unknown models/ids. Usage ===== diff --git a/base_import_odoo/__openerp__.py b/base_import_odoo/__openerp__.py index 5c40fc3a8..31191330d 100644 --- a/base_import_odoo/__openerp__.py +++ b/base_import_odoo/__openerp__.py @@ -13,6 +13,7 @@ ], "demo": [ "demo/res_users.xml", + "demo/ir_attachment.xml", "demo/import_odoo_database.xml", "demo/import_odoo_database_field.xml", "demo/import_odoo_database_model.xml", diff --git a/base_import_odoo/demo/import_odoo_database_field.xml b/base_import_odoo/demo/import_odoo_database_field.xml index af9e3d137..b1a368a27 100644 --- a/base_import_odoo/demo/import_odoo_database_field.xml +++ b/base_import_odoo/demo/import_odoo_database_field.xml @@ -55,5 +55,12 @@ + + + by_reference + + + + diff --git a/base_import_odoo/demo/import_odoo_database_model.xml b/base_import_odoo/demo/import_odoo_database_model.xml index a10fb3bee..bbaa33021 100644 --- a/base_import_odoo/demo/import_odoo_database_model.xml +++ b/base_import_odoo/demo/import_odoo_database_model.xml @@ -19,5 +19,11 @@ [(1, '=', 1)] + + 4 + + + [('res_model', 'in', ['res.users'])] + diff --git a/base_import_odoo/demo/ir_attachment.xml b/base_import_odoo/demo/ir_attachment.xml new file mode 100644 index 000000000..5cfa63277 --- /dev/null +++ b/base_import_odoo/demo/ir_attachment.xml @@ -0,0 +1,11 @@ + + + + + Demo attachment + res.users + + aGVsbG8gd29ybGQK + + + diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py index 2b99d2133..684bf0fbc 100644 --- a/base_import_odoo/models/import_odoo_database.py +++ b/base_import_odoo/models/import_odoo_database.py @@ -393,17 +393,38 @@ class ImportOdooDatabase(models.Model): else: data[field_name] = [(6, 0, data[field_name])] for mapping in self.import_field_mappings: - if mapping.model_id.model != model._name or\ - mapping.mapping_type != 'unique': + if mapping.model_id.model != model._name: continue - for field in mapping.field_ids: - value = data.get(field.name, '') - counter = 1 - while model.with_context(active_test=False).search([ - (field.name, '=', data.get(field.name, value)), - ]): - data[field.name] = '%s (%d)' % (value, counter) - counter += 1 + if mapping.mapping_type == 'unique': + for field in mapping.field_ids: + value = data.get(field.name, '') + counter = 1 + while model.with_context(active_test=False).search([ + (field.name, '=', data.get(field.name, value)), + ]): + data[field.name] = '%s (%d)' % (value, counter) + counter += 1 + elif mapping.mapping_type == 'by_reference': + res_model = data.get(mapping.model_field_id.name) + res_id = data.get(mapping.id_field_id.name) + update = { + mapping.model_field_id.name: None, + mapping.id_field_id.name: None, + } + if res_model in self.env.registry and res_id: + new_context = context.with_field_context( + model._name, res_id, data['id'] + ) + record_id = self._run_import_get_record( + new_context, self.env[res_model], {'id': res_id}, + create_dummy=False + ) + if record_id: + update.update({ + mapping.model_field_id.name: res_model, + mapping.id_field_id.name: record_id, + }) + data.update(update) return data @api.multi diff --git a/base_import_odoo/models/import_odoo_database_field.py b/base_import_odoo/models/import_odoo_database_field.py index 95d9b1959..bec3db204 100644 --- a/base_import_odoo/models/import_odoo_database_field.py +++ b/base_import_odoo/models/import_odoo_database_field.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # © 2017 Therp BV # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). -from openerp import fields, models +from openerp import api, fields, models class ImportOdooDatabaseField(models.Model): @@ -22,6 +22,18 @@ class ImportOdooDatabaseField(models.Model): 'ir.model.fields', string='Field', help='If set, the mapping is only ' 'effective when setting said field', ondelete='cascade', ) + model_field_id = fields.Many2one( + 'ir.model.fields', string='Model field', compute=lambda self: + self._compute_reference_field('model_field_id', 'char'), + inverse=lambda self: + self._inverse_reference_field('model_field_id', 'char'), + ) + id_field_id = fields.Many2one( + 'ir.model.fields', string='ID field', compute=lambda self: + self._compute_reference_field('id_field_id', 'integer'), + inverse=lambda self: + self._inverse_reference_field('id_field_id', 'integer'), + ) # TODO: create a reference function field to set this conveniently local_id = fields.Integer( 'Local ID', help='If you leave this empty, a new record will be ' @@ -36,7 +48,21 @@ class ImportOdooDatabaseField(models.Model): [ ('fixed', 'Fixed'), ('by_field', 'Based on equal fields'), + ('by_reference', 'By reference'), ('unique', 'Unique'), ], string='Type', required=True, default='fixed', ) + + @api.multi + def _compute_reference_field(self, field_name, ttype): + for this in self: + this[field_name] = this.field_ids.filtered( + lambda x: x.ttype == ttype + ) + + @api.multi + def _inverse_reference_field(self, field_name, ttype): + self.field_ids = self.field_ids.filtered( + lambda x: x.ttype != ttype + ) + self[field_name] diff --git a/base_import_odoo/tests/test_base_import_odoo.py b/base_import_odoo/tests/test_base_import_odoo.py index 76739fe56..7951840c5 100644 --- a/base_import_odoo/tests/test_base_import_odoo.py +++ b/base_import_odoo/tests/test_base_import_odoo.py @@ -20,7 +20,7 @@ class TestBaseImportOdoo(TransactionCase): # remote ids are the same def _mock_execute(model, method, *args): if method == 'read': - return self.env[model].browse(args[0]).read(fields=args[1]) + return self.env[model].browse(args[0]).read(args[1]) if method == 'search': return self.env[model].search(args[0]).ids @@ -37,23 +37,19 @@ class TestBaseImportOdoo(TransactionCase): # here the actual test begins - check that we created new # objects, check xmlids, check values, check if dummies are # cleaned up/replaced - self.assertNotEqual( - self.env.ref(self._get_xmlid('base.user_demo')), - self.env.ref('base.user_demo'), - ) + imported_user = self.env.ref(self._get_xmlid('base.user_demo')) + user = self.env.ref('base.user_demo') + self.assertNotEqual(imported_user, user) # check that the imported scalars are equal fields = ['name', 'email', 'signature', 'active'] - ( - self.env.ref(self._get_xmlid('base.user_demo')) + - self.env.ref('base.user_demo') - ).read(fields) + (imported_user + user).read(fields) self.assertEqual( self._get_cache(self._get_xmlid('base.user_demo'), fields), self._get_cache('base.user_demo', fields), ) # check that links are correctly mapped self.assertEqual( - self.env.ref(self._get_xmlid('base.user_demo')).partner_id, + imported_user.partner_id, self.env.ref(self._get_xmlid('base.partner_demo')) ) # no new groups because they should be mapped by name @@ -65,6 +61,15 @@ class TestBaseImportOdoo(TransactionCase): self.env['res.users'].search([], count=True), user_count + (user_count - 1) * run, ) + # check that there's a new attachment + attachment = self.env.ref('base_import_odoo.attachment_demo') + imported_attachment = self.env['ir.attachment'].search([ + ('res_model', '=', 'res.users'), + ('res_id', '=', imported_user.id), + ]) + self.assertTrue(attachment) + self.assertEqual(attachment.datas, imported_attachment.datas) + self.assertNotEqual(attachment, imported_attachment) # TODO: test much more run += 1 demodb = self.env.ref('base_import_odoo.demodb') diff --git a/base_import_odoo/views/import_odoo_database_field.xml b/base_import_odoo/views/import_odoo_database_field.xml index 061b63e50..8d44bd57c 100644 --- a/base_import_odoo/views/import_odoo_database_field.xml +++ b/base_import_odoo/views/import_odoo_database_field.xml @@ -16,13 +16,18 @@
Select fields which must be equal to treat a pair of remote and local records of this model as being equal.
+
+ Select the field that stores the model name and the one that stores the linked ID. The IDs for previously imported records will be mapped to the local ID, for unknown models or IDs, the fields are set to NULL. +
Select fields for which to generate unique values during import. You'll need this for res.users#login for example.
- + + + From 0649f636d592ee848236739d58ec722e2ceb59c8 Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Thu, 12 Oct 2017 10:18:51 +0200 Subject: [PATCH 07/21] [IMP] disable recomputation during import --- base_import_odoo/README.rst | 1 - .../models/import_odoo_database.py | 22 ++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/base_import_odoo/README.rst b/base_import_odoo/README.rst index 3ff3d68b6..96ebe8538 100644 --- a/base_import_odoo/README.rst +++ b/base_import_odoo/README.rst @@ -56,7 +56,6 @@ Known issues / Roadmap * Yes of course this duplicates a lot of connector functionality. Rewrite this with the connector framework, probably collaborate with https://github.com/OCA/connector-odoo2odoo * Do something with workflows * Support reference fields, while being at it refactor _run_import_map_values to call a function per field type -* Probably it's safer and faster to disable recomputation during import, and recompute all fields afterwards Bug Tracker =========== diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py index 684bf0fbc..0ffda50b1 100644 --- a/base_import_odoo/models/import_odoo_database.py +++ b/base_import_odoo/models/import_odoo_database.py @@ -160,6 +160,7 @@ class ImportOdooDatabase(models.Model): """Import records of a configured model""" model = self.env[context.model_line.model_id.model] fields = self._run_import_model_get_fields(context) + recompute_ids = [] for data in context.remote.execute( model._name, 'read', context.ids, fields.keys() ): @@ -170,9 +171,16 @@ class ImportOdooDatabase(models.Model): data = self._run_import_map_values(context, data) _id = data['id'] record = self._create_record(context, model, data) + recompute_ids.append(record.id) self._run_import_model_cleanup_dummies( context, model, _id, record.id, ) + to_recompute = model.browse(recompute_ids) + for field in model._fields.values(): + if not field.compute: + continue + to_recompute._recompute_todo(field) + to_recompute.recompute() @api.multi def _create_record(self, context, model, record): @@ -183,13 +191,15 @@ class ImportOdooDatabase(models.Model): ) if self.env.ref('base_import_odoo.%s' % xmlid, False): new = self.env.ref('base_import_odoo.%s' % xmlid) - new.with_context( - **self._create_record_context(model, record) - ).write(record) + with self.env.norecompute(): + new.with_context( + **self._create_record_context(model, record) + ).write(record) else: - new = model.with_context( - **self._create_record_context(model, record) - ).create(record) + with self.env.norecompute(): + new = model.with_context( + **self._create_record_context(model, record) + ).create(record) self.env['ir.model.data'].create({ 'name': xmlid, 'model': model._name, From c6744dea42873ad553ed2699b4bca44260aea5ba Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Thu, 12 Oct 2017 11:34:30 +0200 Subject: [PATCH 08/21] [ADD] better progress --- .../tests/test_base_import_odoo.py | 2 ++ .../views/import_odoo_database.xml | 24 +++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/base_import_odoo/tests/test_base_import_odoo.py b/base_import_odoo/tests/test_base_import_odoo.py index 7951840c5..8b3128306 100644 --- a/base_import_odoo/tests/test_base_import_odoo.py +++ b/base_import_odoo/tests/test_base_import_odoo.py @@ -73,6 +73,8 @@ class TestBaseImportOdoo(TransactionCase): # TODO: test much more run += 1 demodb = self.env.ref('base_import_odoo.demodb') + for line in demodb.import_line_ids: + self.assertIn(line.model_id.model, demodb.status_html) demodb.action_import() self.assertTrue(demodb.cronjob_id) demodb.cronjob_id.write({'active': False}) diff --git a/base_import_odoo/views/import_odoo_database.xml b/base_import_odoo/views/import_odoo_database.xml index e31548d93..c919798c9 100644 --- a/base_import_odoo/views/import_odoo_database.xml +++ b/base_import_odoo/views/import_odoo_database.xml @@ -20,13 +20,13 @@ + - @@ -64,17 +64,17 @@ }); } -
- - -
-
- - / done - -
- -
+

Import progress

+
+
+ + +

+ + / done + +

+ 

         
     

From 46b6e03c3d69b9dc3d2c1b298be6312f07467266 Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Thu, 12 Oct 2017 11:50:10 +0200
Subject: [PATCH 09/21] Revert "[IMP] disable recomputation during import"

This reverts commit efce253d54e08519039198aea7fcc93fad263055.
---
 base_import_odoo/README.rst                   |  1 +
 .../models/import_odoo_database.py            | 22 +++++--------------
 2 files changed, 7 insertions(+), 16 deletions(-)

diff --git a/base_import_odoo/README.rst b/base_import_odoo/README.rst
index 96ebe8538..3ff3d68b6 100644
--- a/base_import_odoo/README.rst
+++ b/base_import_odoo/README.rst
@@ -56,6 +56,7 @@ Known issues / Roadmap
 * Yes of course this duplicates a lot of connector functionality. Rewrite this with the connector framework, probably collaborate with https://github.com/OCA/connector-odoo2odoo
 * Do something with workflows
 * Support reference fields, while being at it refactor _run_import_map_values to call a function per field type
+* Probably it's safer and faster to disable recomputation during import, and recompute all fields afterwards
 
 Bug Tracker
 ===========
diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py
index 0ffda50b1..684bf0fbc 100644
--- a/base_import_odoo/models/import_odoo_database.py
+++ b/base_import_odoo/models/import_odoo_database.py
@@ -160,7 +160,6 @@ class ImportOdooDatabase(models.Model):
         """Import records of a configured model"""
         model = self.env[context.model_line.model_id.model]
         fields = self._run_import_model_get_fields(context)
-        recompute_ids = []
         for data in context.remote.execute(
                 model._name, 'read', context.ids, fields.keys()
         ):
@@ -171,16 +170,9 @@ class ImportOdooDatabase(models.Model):
             data = self._run_import_map_values(context, data)
             _id = data['id']
             record = self._create_record(context, model, data)
-            recompute_ids.append(record.id)
             self._run_import_model_cleanup_dummies(
                 context, model, _id, record.id,
             )
-        to_recompute = model.browse(recompute_ids)
-        for field in model._fields.values():
-            if not field.compute:
-                continue
-            to_recompute._recompute_todo(field)
-        to_recompute.recompute()
 
     @api.multi
     def _create_record(self, context, model, record):
@@ -191,15 +183,13 @@ class ImportOdooDatabase(models.Model):
         )
         if self.env.ref('base_import_odoo.%s' % xmlid, False):
             new = self.env.ref('base_import_odoo.%s' % xmlid)
-            with self.env.norecompute():
-                new.with_context(
-                    **self._create_record_context(model, record)
-                ).write(record)
+            new.with_context(
+                **self._create_record_context(model, record)
+            ).write(record)
         else:
-            with self.env.norecompute():
-                new = model.with_context(
-                    **self._create_record_context(model, record)
-                ).create(record)
+            new = model.with_context(
+                **self._create_record_context(model, record)
+            ).create(record)
             self.env['ir.model.data'].create({
                 'name': xmlid,
                 'model': model._name,

From 8d91dd949180493e1efca1ed8520ee5a95523597 Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Thu, 12 Oct 2017 12:37:41 +0200
Subject: [PATCH 10/21] [ADD] more tests

---
 base_import_odoo/tests/test_base_import_odoo.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/base_import_odoo/tests/test_base_import_odoo.py b/base_import_odoo/tests/test_base_import_odoo.py
index 8b3128306..744350fc1 100644
--- a/base_import_odoo/tests/test_base_import_odoo.py
+++ b/base_import_odoo/tests/test_base_import_odoo.py
@@ -3,6 +3,7 @@
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 from mock import patch
 from openerp.tests.common import TransactionCase, post_install, at_install
+from ..models.import_odoo_database import ImportContext, field_context
 
 
 class TestBaseImportOdoo(TransactionCase):
@@ -70,7 +71,6 @@ class TestBaseImportOdoo(TransactionCase):
             self.assertTrue(attachment)
             self.assertEqual(attachment.datas, imported_attachment.datas)
             self.assertNotEqual(attachment, imported_attachment)
-            # TODO: test much more
             run += 1
         demodb = self.env.ref('base_import_odoo.demodb')
         for line in demodb.import_line_ids:
@@ -81,6 +81,15 @@ class TestBaseImportOdoo(TransactionCase):
         demodb.action_import()
         self.assertTrue(demodb.cronjob_id.active)
         self.assertFalse(demodb.cronjob_running)
+        # in our setting we won't get dummies, so we test this manually
+        import_context = ImportContext(
+            None, None, [], {}, {}, [], {}, field_context(None, None, None)
+        )
+        dummy_id = demodb._run_import_create_dummy(
+            import_context, self.env['res.partner'], {'id': 424242},
+            forcecreate=True,
+        )
+        self.assertTrue(self.env['res.partner'].browse(dummy_id).exists())
 
     def _get_xmlid(self, remote_xmlid):
         remote_obj = self.env.ref(remote_xmlid)

From ddb851f7247ffa36f513f92f2c4f58042266f6dd Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Fri, 13 Oct 2017 02:48:29 +0200
Subject: [PATCH 11/21] [FIX] dummy cleanup

[ADD] debug logging
---
 base_import_odoo/__openerp__.py               |   1 +
 base_import_odoo/demo/res_partner.xml         |   8 ++
 .../models/import_odoo_database.py            | 107 +++++++++++++++---
 .../views/import_odoo_database.xml            |  10 ++
 4 files changed, 109 insertions(+), 17 deletions(-)
 create mode 100644 base_import_odoo/demo/res_partner.xml

diff --git a/base_import_odoo/__openerp__.py b/base_import_odoo/__openerp__.py
index 31191330d..fdd6f8f7f 100644
--- a/base_import_odoo/__openerp__.py
+++ b/base_import_odoo/__openerp__.py
@@ -12,6 +12,7 @@
         'base',
     ],
     "demo": [
+        "demo/res_partner.xml",
         "demo/res_users.xml",
         "demo/ir_attachment.xml",
         "demo/import_odoo_database.xml",
diff --git a/base_import_odoo/demo/res_partner.xml b/base_import_odoo/demo/res_partner.xml
new file mode 100644
index 000000000..50c1b2a99
--- /dev/null
+++ b/base_import_odoo/demo/res_partner.xml
@@ -0,0 +1,8 @@
+
+
+    
+        
+            
+        
+    
+
diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py
index 684bf0fbc..f35101f0b 100644
--- a/base_import_odoo/models/import_odoo_database.py
+++ b/base_import_odoo/models/import_odoo_database.py
@@ -11,6 +11,7 @@ import traceback
 from urlparse import urlparse
 from openerp import _, api, exceptions, fields, models, tools
 from collections import namedtuple
+_logger = logging.getLogger('base_import_odoo')
 
 
 import_context_tuple = namedtuple(
@@ -116,6 +117,7 @@ class ImportOdooDatabase(models.Model):
                 'counts': remote_counts,
                 'ids': remote_ids,
                 'error': None,
+                'dummies': None,
                 'done': {},
             }
         })
@@ -151,9 +153,16 @@ class ImportOdooDatabase(models.Model):
                     raise
                 done[model._name] += len(ids)
                 self.write({'status_data': dict(self.status_data, done=done)})
+
                 if commit and not tools.config['test_enable']:
                     # pylint: disable=invalid-commit
                     self.env.cr.commit()
+        missing = {}
+        for dummy_model, remote_id in dummies.keys():
+            missing.setdefault(dummy_model, []).append(remote_id)
+        self.write({
+            'status_data': dict(self.status_data, dummies=dict(missing)),
+        })
 
     @api.multi
     def _run_import_model(self, context):
@@ -163,7 +172,9 @@ class ImportOdooDatabase(models.Model):
         for data in context.remote.execute(
                 model._name, 'read', context.ids, fields.keys()
         ):
-            self._run_import_get_record(context, model, data)
+            self._run_import_get_record(
+                context, model, data, create_dummy=False,
+            )
             if (model._name, data['id']) in context.idmap:
                 # there's a mapping for this record, nothing to do
                 continue
@@ -179,13 +190,14 @@ class ImportOdooDatabase(models.Model):
         """Create a record, add an xmlid"""
         _id = record.pop('id')
         xmlid = '%d-%s-%d' % (
-            self.id, model._name.replace('.', '_'), _id,
+            self.id, model._name.replace('.', '_'), _id or 0,
         )
         if self.env.ref('base_import_odoo.%s' % xmlid, False):
             new = self.env.ref('base_import_odoo.%s' % xmlid)
             new.with_context(
                 **self._create_record_context(model, record)
             ).write(record)
+            _logger.debug('Updated record %s', xmlid)
         else:
             new = model.with_context(
                 **self._create_record_context(model, record)
@@ -199,6 +211,7 @@ class ImportOdooDatabase(models.Model):
                 'import_database_id': self.id,
                 'import_database_record_id': _id,
             })
+            _logger.debug('Created record %s', xmlid)
         context.idmap[mapping_key(model._name, _id)] = new.id
         return new
 
@@ -218,16 +231,28 @@ class ImportOdooDatabase(models.Model):
         """Find the local id of some remote record. Create a dummy if not
         available"""
         _id = context.idmap.get((model._name, record['id']))
+        logged = False
         if not _id:
             _id = context.dummies.get((model._name, record['id']))
             if _id:
                 context.dummy_instances.append(
                     dummy_instance(*(context.field_context + (_id,)))
                 )
+        else:
+            logged = True
+            _logger.debug(
+                'Got %s(%d[%d]) from idmap', model._model, _id,
+                record['id'] or 0,
+            )
         if not _id:
             _id = self._run_import_get_record_mapping(
                 context, model, record, create_dummy=create_dummy,
             )
+        elif not logged:
+            logged = True
+            _logger.debug(
+                'Got %s(%d[%d]) from dummies', model._model, _id, record['id'],
+            )
         if not _id:
             xmlid = self.env['ir.model.data'].search([
                 ('import_database_id', '=', self.id),
@@ -237,8 +262,22 @@ class ImportOdooDatabase(models.Model):
             if xmlid:
                 _id = xmlid.res_id
                 context.idmap[(model._name, record['id'])] = _id
+        elif not logged:
+            logged = True
+            _logger.debug(
+                'Got %s(%d[%d]) from mappings',
+                model._model, _id, record['id'],
+            )
         if not _id and create_dummy:
-            _id = self._run_import_create_dummy(context, model, record)
+            _id = self._run_import_create_dummy(
+                context, model, record,
+                forcecreate=record['id'] not in
+                self.status_data['ids'].get(model._name, [])
+            )
+        elif _id and not logged:
+            _logger.debug(
+                'Got %s(%d[%d]) from xmlid', model._model, _id, record['id'],
+            )
         return _id
 
     @api.multi
@@ -270,9 +309,19 @@ class ImportOdooDatabase(models.Model):
             elif mapping.mapping_type == 'by_field':
                 assert mapping.field_ids
                 if len(record) == 1:
-                    continue
+                    # just the id of a record we haven't seen yet.
+                    # read the whole record from remote to check if
+                    # this can be mapped to an existing record
+                    record = context.remote.execute(
+                        model._name, 'read', record['id'],
+                        mapping.field_ids.mapped('name'),
+                    ) or None
+                    if not record:
+                        continue
+                    if isinstance(record, list):
+                        record = record[0]
                 records = model.search([
-                    (field.name, '=', record[field.name])
+                    (field.name, '=', record.get(field.name))
                     for field in mapping.field_ids
                 ], limit=1)
                 if records:
@@ -310,6 +359,13 @@ class ImportOdooDatabase(models.Model):
             context.dummy_instances.append(
                 dummy_instance(*(context.field_context + (dummy.id,)))
             )
+            _logger.debug(
+                'Using %d as dummy for %s(%d[%d]).%s[%d]',
+                dummy.id, context.field_context.record_model,
+                context.idmap.get(context.field_context.record_id, 0),
+                context.field_context.record_id,
+                context.field_context.field_name, record['id'],
+            )
             return dummy.id
         required = [
             name
@@ -331,6 +387,8 @@ class ImportOdooDatabase(models.Model):
             elif model._fields[name].type in ['date', 'datetime']:
                 value = '2000-01-01'
             elif field.type in ['many2one']:
+                if name in model._inherits.values():
+                    continue
                 new_context = context.with_field_context(
                     model._name, name, record['id']
                 )
@@ -345,12 +403,20 @@ class ImportOdooDatabase(models.Model):
                 value = field.selection(model)[0][0]
             values[name] = value
         dummy = self._create_record(context, model, values)
+        del context.idmap[mapping_key(model._name, record['id'])]
         context.dummies[mapping_key(model._name, record['id'])] = dummy.id
         context.to_delete.setdefault(model._name, [])
         context.to_delete[model._name].append(dummy.id)
         context.dummy_instances.append(
             dummy_instance(*(context.field_context + (dummy.id,)))
         )
+        _logger.debug(
+            'Created %d as dummy for %s(%d[%d]).%s[%d]',
+            dummy.id, context.field_context.record_model,
+            context.idmap.get(context.field_context.record_id, 0),
+            context.field_context.record_id or 0,
+            context.field_context.field_name, record['id'],
+        )
         return dummy.id
 
     @api.multi
@@ -378,7 +444,7 @@ class ImportOdooDatabase(models.Model):
                     new_context, comodel, {'id': _id},
                     create_dummy=model._fields[field_name].required or
                     any(
-                        m.model_id._name == comodel._name
+                        m.model_id.model == comodel._name
                         for m in self.import_line_ids
                     ),
                 )
@@ -440,22 +506,28 @@ class ImportOdooDatabase(models.Model):
     def _run_import_model_cleanup_dummies(
             self, context, model, remote_id, local_id
     ):
+        if not (model._name, remote_id) in context.dummies:
+            return
         for instance in context.dummy_instances:
-            if (
-                    instance.model_name != model._name or
-                    instance.remote_id != remote_id
-            ):
+            key = mapping_key(instance.model_name, instance.remote_id)
+            if key not in context.idmap:
                 continue
-            if not context.idmap.get(instance.remote_id):
+            dummy_id = context.dummies[(model._name, remote_id)]
+            record_model = self.env[instance.model_name]
+            comodel = record_model._fields[instance.field_name].comodel_name
+            if comodel != model._name or instance.dummy_id != dummy_id:
                 continue
-            model = self.env[instance.model_name]
-            record = model.browse(context.idmap[instance.remote_id])
-            field_name = instance.field_id.name
+            record = record_model.browse(context.idmap[key])
+            field_name = instance.field_name
+            _logger.debug(
+                'Replacing dummy %d on %s(%d).%s with %d',
+                dummy_id, record_model._name, record.id, field_name, local_id,
+            )
             if record._fields[field_name].type == 'many2one':
                 record.write({field_name: local_id})
             elif record._fields[field_name].type == 'many2many':
                 record.write({field_name: [
-                    (3, context.idmap[remote_id]),
+                    (3, dummy_id),
                     (4, local_id),
                 ]})
             else:
@@ -464,10 +536,11 @@ class ImportOdooDatabase(models.Model):
                     record._fields[field_name].type
                 )
             context.dummy_instances.remove(instance)
-            dummy_id = context.dummies[(record._model, remote_id)]
             if dummy_id in context.to_delete:
                 model.browse(dummy_id).unlink()
-            del context.dummies[(record._model, remote_id)]
+                _logger.debug('Deleting dummy %d', dummy_id)
+        if (model._name, remote_id) in context.dummies:
+            del context.dummies[(model._name, remote_id)]
 
     def _get_connection(self):
         self.ensure_one()
diff --git a/base_import_odoo/views/import_odoo_database.xml b/base_import_odoo/views/import_odoo_database.xml
index c919798c9..ab8333a8d 100644
--- a/base_import_odoo/views/import_odoo_database.xml
+++ b/base_import_odoo/views/import_odoo_database.xml
@@ -76,6 +76,16 @@
                 
             
             

+            

+                The following remote ids don't have a mapping but have to be imported anyways due to not null constraints.
+                

+                    
+                        

+                        

+                    
+                

+                To fix this, create mappings for the remote ids listed, or if this is not feasible, map the whole model. You might also have a too specific domain on your import model definition.
+            

         
     
 

From 37cbc2ac1b469b83291ee76aba19e3cb43b2742c Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Tue, 17 Oct 2017 08:25:29 +0200
Subject: [PATCH 12/21] [ADD] depend on mail

---
 base_import_odoo/__openerp__.py                      |  2 +-
 base_import_odoo/demo/import_odoo_database_field.xml |  6 ++++++
 base_import_odoo/demo/import_odoo_database_model.xml | 12 +++++++++---
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/base_import_odoo/__openerp__.py b/base_import_odoo/__openerp__.py
index fdd6f8f7f..f58f96a07 100644
--- a/base_import_odoo/__openerp__.py
+++ b/base_import_odoo/__openerp__.py
@@ -9,7 +9,7 @@
     "category": "Tools",
     "summary": "Import records from another Odoo instance",
     "depends": [
-        'base',
+        'mail',
     ],
     "demo": [
         "demo/res_partner.xml",
diff --git a/base_import_odoo/demo/import_odoo_database_field.xml b/base_import_odoo/demo/import_odoo_database_field.xml
index b1a368a27..cb817cabe 100644
--- a/base_import_odoo/demo/import_odoo_database_field.xml
+++ b/base_import_odoo/demo/import_odoo_database_field.xml
@@ -49,6 +49,12 @@
             
             
         
+        
+            
+            by_field
+            
+            
+        
         
             
             by_field
diff --git a/base_import_odoo/demo/import_odoo_database_model.xml b/base_import_odoo/demo/import_odoo_database_model.xml
index bbaa33021..1b07dc7a9 100644
--- a/base_import_odoo/demo/import_odoo_database_model.xml
+++ b/base_import_odoo/demo/import_odoo_database_model.xml
@@ -7,20 +7,26 @@
             
             [(1, '=', 1)]
         
-        
+        
             2
+            
+            
+            [(1, '=', 1)]
+        
+        
+            3
             
             
             [(1, '=', 1)]
         
         
-            3
+            4
             
             
             [(1, '=', 1)]
         
         
-            4
+            5
             
             
             [('res_model', 'in', ['res.users'])]

From 1b25d70fc592fd7e2a06c17f119113096654464d Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Tue, 17 Oct 2017 19:34:33 +0200
Subject: [PATCH 13/21] fixup! [FIX] dummy cleanup

---
 base_import_odoo/models/import_odoo_database.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py
index f35101f0b..3965dd49f 100644
--- a/base_import_odoo/models/import_odoo_database.py
+++ b/base_import_odoo/models/import_odoo_database.py
@@ -159,7 +159,8 @@ class ImportOdooDatabase(models.Model):
                     self.env.cr.commit()
         missing = {}
         for dummy_model, remote_id in dummies.keys():
-            missing.setdefault(dummy_model, []).append(remote_id)
+            if remote_id:
+                missing.setdefault(dummy_model, []).append(remote_id)
         self.write({
             'status_data': dict(self.status_data, dummies=dict(missing)),
         })

From e3feaf1c0957865d47d2c0842db32dfea2406831 Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Thu, 19 Oct 2017 15:10:43 +0200
Subject: [PATCH 14/21] [ADD] fix when running tests with accounting

---
 .../tests/test_base_import_odoo.py            | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/base_import_odoo/tests/test_base_import_odoo.py b/base_import_odoo/tests/test_base_import_odoo.py
index 744350fc1..cd04c3988 100644
--- a/base_import_odoo/tests/test_base_import_odoo.py
+++ b/base_import_odoo/tests/test_base_import_odoo.py
@@ -7,6 +7,27 @@ from ..models.import_odoo_database import ImportContext, field_context
 
 
 class TestBaseImportOdoo(TransactionCase):
+    def setUp(self):
+        super(TestBaseImportOdoo, self).setUp()
+        # if our tests run with an accounting scheme, it will fail on accounts
+        # to fix this, if the account model exists, we create mappings for it
+        if 'account.account' in self.env.registry:
+            self.env.ref('base_import_odoo.demodb').write({
+                'import_field_mappings': [
+                    (
+                        4,
+                        {
+                            'mapping_type': 'fixed',
+                            'model_id':
+                            self.env.ref('account.model_account_account').id,
+                            'local_id': account.id,
+                            'remote_id': account.id,
+                        },
+                    )
+                    for account in self.env['account.account'].search([])
+                ],
+            })
+
     @at_install(False)
     @post_install(True)
     @patch(

From e89f8ee45acb01b680ac6e824a7b03fc81a01f34 Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Thu, 19 Oct 2017 16:16:56 +0200
Subject: [PATCH 15/21] fixup! [ADD] fix when running tests with accounting

---
 base_import_odoo/tests/test_base_import_odoo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base_import_odoo/tests/test_base_import_odoo.py b/base_import_odoo/tests/test_base_import_odoo.py
index cd04c3988..c1e9c7511 100644
--- a/base_import_odoo/tests/test_base_import_odoo.py
+++ b/base_import_odoo/tests/test_base_import_odoo.py
@@ -15,7 +15,7 @@ class TestBaseImportOdoo(TransactionCase):
             self.env.ref('base_import_odoo.demodb').write({
                 'import_field_mappings': [
                     (
-                        4,
+                        0, False,
                         {
                             'mapping_type': 'fixed',
                             'model_id':

From 8a739e4e81a16453896cfd6e9424c8cc98a87af8 Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Thu, 19 Oct 2017 16:23:16 +0200
Subject: [PATCH 16/21] [ADD] duplicate treatment

---
 base_import_odoo/README.rst                   |  3 ++-
 .../models/import_odoo_database.py            | 20 ++++++++++++++++---
 .../views/import_odoo_database.xml            |  6 ++++--
 3 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/base_import_odoo/README.rst b/base_import_odoo/README.rst
index 3ff3d68b6..af6385e2e 100644
--- a/base_import_odoo/README.rst
+++ b/base_import_odoo/README.rst
@@ -12,7 +12,7 @@ Use cases
 =========
 
 - merging databases
-- one way sync (needs a bit polishing)
+- one way sync
 - aggregating management data from distributed systems
 
 
@@ -57,6 +57,7 @@ Known issues / Roadmap
 * Do something with workflows
 * Support reference fields, while being at it refactor _run_import_map_values to call a function per field type
 * Probably it's safer and faster to disable recomputation during import, and recompute all fields afterwards
+* Add duplicate handling strategy 'Overwrite older'
 
 Bug Tracker
 ===========
diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py
index 3965dd49f..2d69cbcaa 100644
--- a/base_import_odoo/models/import_odoo_database.py
+++ b/base_import_odoo/models/import_odoo_database.py
@@ -62,6 +62,13 @@ class ImportOdooDatabase(models.Model):
     status_html = fields.Html(
         compute='_compute_status_html', readonly=True, sanitize=False,
     )
+    duplicates = fields.Selection(
+        [
+            ('skip', 'Skip existing'), ('overwrite', 'Overwrite existing'),
+            ('overwrite_empty', 'Overwrite empty fields'),
+        ],
+        'Duplicate handling', default='skip', required=True,
+    )
 
     @api.multi
     def action_import(self):
@@ -177,8 +184,9 @@ class ImportOdooDatabase(models.Model):
                 context, model, data, create_dummy=False,
             )
             if (model._name, data['id']) in context.idmap:
-                # there's a mapping for this record, nothing to do
-                continue
+                if self.duplicates == 'skip':
+                    # there's a mapping for this record, nothing to do
+                    continue
             data = self._run_import_map_values(context, data)
             _id = data['id']
             record = self._create_record(context, model, data)
@@ -193,8 +201,14 @@ class ImportOdooDatabase(models.Model):
         xmlid = '%d-%s-%d' % (
             self.id, model._name.replace('.', '_'), _id or 0,
         )
-        if self.env.ref('base_import_odoo.%s' % xmlid, False):
+        if self.env.ref('base_import_odoo.%s' % xmlid, False).exists():
             new = self.env.ref('base_import_odoo.%s' % xmlid)
+            if self.duplicates == 'overwrite_empty':
+                record = {
+                    key: value
+                    for key, value in record.items()
+                    if not new[key]
+                }
             new.with_context(
                 **self._create_record_context(model, record)
             ).write(record)
diff --git a/base_import_odoo/views/import_odoo_database.xml b/base_import_odoo/views/import_odoo_database.xml
index ab8333a8d..a67e8c823 100644
--- a/base_import_odoo/views/import_odoo_database.xml
+++ b/base_import_odoo/views/import_odoo_database.xml
@@ -26,6 +26,7 @@
                             
                             
                             
+                            
                             
                         
                         
@@ -64,8 +65,9 @@
                     });
                 }
             
-            
Import progress

-            

+            
Import progress

+            
Import results

+            

                 

                     
                     

From c4a9054ce3135589f239d05a79e3c260a6336cff Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Thu, 19 Oct 2017 16:46:27 +0200
Subject: [PATCH 17/21] fixup! [ADD] duplicate treatment

---
 base_import_odoo/models/import_odoo_database.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py
index 2d69cbcaa..9c3d10188 100644
--- a/base_import_odoo/models/import_odoo_database.py
+++ b/base_import_odoo/models/import_odoo_database.py
@@ -201,8 +201,8 @@ class ImportOdooDatabase(models.Model):
         xmlid = '%d-%s-%d' % (
             self.id, model._name.replace('.', '_'), _id or 0,
         )
-        if self.env.ref('base_import_odoo.%s' % xmlid, False).exists():
-            new = self.env.ref('base_import_odoo.%s' % xmlid)
+        new = self.env.ref('base_import_odoo.%s' % xmlid, False)
+        if new and new.exists():
             if self.duplicates == 'overwrite_empty':
                 record = {
                     key: value

From 06be3271dade74c1ee7d95214a26bdd300fe9a45 Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Tue, 24 Oct 2017 16:22:19 +0200
Subject: [PATCH 18/21] fixup! [ADD] duplicate treatment

---
 base_import_odoo/models/import_odoo_database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py
index 9c3d10188..c8d8ff079 100644
--- a/base_import_odoo/models/import_odoo_database.py
+++ b/base_import_odoo/models/import_odoo_database.py
@@ -335,7 +335,7 @@ class ImportOdooDatabase(models.Model):
                         continue
                     if isinstance(record, list):
                         record = record[0]
-                records = model.search([
+                records = model.with_context(active_test=False).search([
                     (field.name, '=', record.get(field.name))
                     for field in mapping.field_ids
                 ], limit=1)

From a22452a71666c015d2d77263c54f302564966818 Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Tue, 24 Oct 2017 19:28:55 +0200
Subject: [PATCH 19/21] fixup! [ADD] duplicate treatment

---
 base_import_odoo/models/import_odoo_database.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py
index c8d8ff079..6d78f80ca 100644
--- a/base_import_odoo/models/import_odoo_database.py
+++ b/base_import_odoo/models/import_odoo_database.py
@@ -335,10 +335,18 @@ class ImportOdooDatabase(models.Model):
                         continue
                     if isinstance(record, list):
                         record = record[0]
-                records = model.with_context(active_test=False).search([
+                domain = [
                     (field.name, '=', record.get(field.name))
                     for field in mapping.field_ids
-                ], limit=1)
+                    if record.get(field.name)
+                ]
+                if len(domain) < len(mapping.field_ids):
+                    # play it save, only use mapping if we really select
+                    # something specific
+                    continue
+                records = model.with_context(active_test=False).search(
+                    domain, limit=1,
+                )
                 if records:
                     _id = records.id
                     context.idmap[(model._name, record['id'])] = _id

From 787aa4bcf742f93ea98d611d6e71bf6285d31c99 Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Wed, 25 Oct 2017 08:07:00 +0200
Subject: [PATCH 20/21] [FIX] don't pass an empty string for required char
 fields

---
 base_import_odoo/models/import_odoo_database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py
index 6d78f80ca..964dc9670 100644
--- a/base_import_odoo/models/import_odoo_database.py
+++ b/base_import_odoo/models/import_odoo_database.py
@@ -402,7 +402,7 @@ class ImportOdooDatabase(models.Model):
                 continue
             value = None
             if field.type in ['char', 'text', 'html']:
-                value = ''
+                value = '/'
             elif field.type in ['boolean']:
                 value = False
             elif field.type in ['integer', 'float']:

From 049d2cb8f4d1d5ce9b8e4adf02b1075834a5c57d Mon Sep 17 00:00:00 2001
From: Holger Brunn 
Date: Wed, 25 Oct 2017 09:20:28 +0200
Subject: [PATCH 21/21] fixup! [FIX] don't pass an empty string for required
 char fields

---
 base_import_odoo/models/import_odoo_database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base_import_odoo/models/import_odoo_database.py b/base_import_odoo/models/import_odoo_database.py
index 964dc9670..f8e750a62 100644
--- a/base_import_odoo/models/import_odoo_database.py
+++ b/base_import_odoo/models/import_odoo_database.py
@@ -398,7 +398,7 @@ class ImportOdooDatabase(models.Model):
         defaults = model.default_get(required)
         values = {'id': record['id']}
         for name, field in model._fields.items():
-            if name not in required or name in defaults:
+            if name not in required or defaults.get(name):
                 continue
             value = None
             if field.type in ['char', 'text', 'html']: