From 29c0fa2ce01e0329403e54990fc7c34a0986628a Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Wed, 19 Apr 2017 19:56:11 +0200 Subject: [PATCH] [ADD] allow creating missing indexes and purging properties (#736) * [ADD] allow creating missing indexes * [FIX] tests; installation * [ADD] allow purging properties * [ADD] missing file * [ADD] test purging properties * [ADD] missing parent_id for menu entry * [FIX] don't delete too many and wrong properties --- database_cleanup/__manifest__.py | 2 + database_cleanup/models/__init__.py | 2 + database_cleanup/models/create_indexes.py | 80 +++++++++++++ database_cleanup/models/purge_properties.py | 110 ++++++++++++++++++ .../tests/test_database_cleanup.py | 15 +++ database_cleanup/views/create_indexes.xml | 52 +++++++++ database_cleanup/views/menu.xml | 14 +++ database_cleanup/views/purge_properties.xml | 49 ++++++++ database_cleanup/views/purge_wizard.xml | 7 +- 9 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 database_cleanup/models/create_indexes.py create mode 100644 database_cleanup/models/purge_properties.py create mode 100644 database_cleanup/views/create_indexes.xml create mode 100644 database_cleanup/views/purge_properties.xml diff --git a/database_cleanup/__manifest__.py b/database_cleanup/__manifest__.py index 036988cf5..b9ce75744 100644 --- a/database_cleanup/__manifest__.py +++ b/database_cleanup/__manifest__.py @@ -16,6 +16,8 @@ 'views/purge_columns.xml', 'views/purge_tables.xml', 'views/purge_data.xml', + "views/create_indexes.xml", + 'views/purge_properties.xml', 'views/menu.xml', ], 'installable': False, diff --git a/database_cleanup/models/__init__.py b/database_cleanup/models/__init__.py index 1857ee3b0..f9de433cf 100644 --- a/database_cleanup/models/__init__.py +++ b/database_cleanup/models/__init__.py @@ -6,3 +6,5 @@ from . import purge_tables from . import purge_data from . import purge_menus from . import ir_model_fields +from . import create_indexes +from . import purge_properties diff --git a/database_cleanup/models/create_indexes.py b/database_cleanup/models/create_indexes.py new file mode 100644 index 000000000..ab98d5c2a --- /dev/null +++ b/database_cleanup/models/create_indexes.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from ..identifier_adapter import IdentifierAdapter +from openerp import api, fields, models + + +class CreateIndexesLine(models.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.create_indexes.line' + + purged = fields.Boolean('Created') + wizard_id = fields.Many2one('cleanup.create_indexes.wizard') + field_id = fields.Many2one('ir.model.fields', required=True) + + @api.multi + def purge(self): + tables = set() + for field in self.mapped('field_id'): + model = self.env[field.model] + name = '%s_%s_index' % (model._table, field.name) + self.env.cr.execute( + 'create index %s ON %s (%s)', + ( + IdentifierAdapter(name, quote=False), + IdentifierAdapter(model._table), + IdentifierAdapter(field.name), + ), + ) + tables.add(model._table) + for table in tables: + self.env.cr.execute( + 'analyze %s', (IdentifierAdapter(model._table),) + ) + self.write({ + 'purged': True, + }) + + +class CreateIndexesWizard(models.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.create_indexes.wizard' + _description = 'Create indexes' + + purge_line_ids = fields.One2many( + 'cleanup.create_indexes.line', 'wizard_id', + ) + + @api.multi + def find(self): + for field in self.env['ir.model.fields'].search([ + ('index', '=', True), + ]): + if field.model not in self.env.registry: + continue + model = self.env[field.model] + name = '%s_%s_index' % (model._table, field.name) + self.env.cr.execute( + 'select indexname from pg_indexes ' + 'where indexname=%s and tablename=%s', + (name, model._table) + ) + if self.env.cr.rowcount: + continue + + self.env.cr.execute( + 'select a.attname ' + 'from pg_attribute a ' + 'join pg_class c on a.attrelid=c.oid ' + 'join pg_tables t on t.tablename=c.relname ' + 'where attname=%s and c.relname=%s', + (field.name, model._table,) + ) + if not self.env.cr.rowcount: + continue + + yield (0, 0, { + 'name': '%s.%s' % (field.model, field.name), + 'field_id': field.id, + }) diff --git a/database_cleanup/models/purge_properties.py b/database_cleanup/models/purge_properties.py new file mode 100644 index 000000000..bd1f44ac3 --- /dev/null +++ b/database_cleanup/models/purge_properties.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from openerp import api, models, fields +REASON_DUPLICATE = 1 +REASON_DEFAULT = 2 + + +class CleanupPurgeLineProperty(models.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.property' + _description = 'Purge properties' + + wizard_id = fields.Many2one( + 'cleanup.purge.wizard.property', 'Purge Wizard', readonly=True) + property_id = fields.Many2one('ir.property') + reason = fields.Selection([ + (REASON_DUPLICATE, 'Duplicated property'), + (REASON_DEFAULT, 'Same value as default'), + ]) + + @api.multi + def purge(self): + """Delete properties""" + self.write({'purged': True}) + return self.mapped('property_id').unlink() + + +class CleanupPurgeWizardProperty(models.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.property' + _description = 'Purge properties' + + @api.model + def find(self): + """ + Search property records which are duplicated or the same as the default + """ + result = [] + default_properties = self.env['ir.property'].search([ + ('res_id', '=', False), + ]) + handled_field_ids = [] + for prop in default_properties: + if prop.fields_id.id in handled_field_ids: + continue + domain = [ + ('id', '!=', prop.id), + ('fields_id', '=', prop.fields_id.id), + # =? explicitly tests for None or False, not falsyness + ('value_float', '=?', prop.value_float or False), + ('value_integer', '=?', prop.value_integer or False), + ('value_text', '=?', prop.value_text or False), + ('value_binary', '=?', prop.value_binary or False), + ('value_reference', '=?', prop.value_reference or False), + ('value_datetime', '=?', prop.value_datetime or False), + ] + if prop.company_id: + domain.append(('company_id', '=', prop.company_id.id)) + else: + domain.extend([ + '|', + ('company_id', '=', False), + ( + 'company_id', 'in', self.env['res.company'].search([ + ( + 'id', 'not in', default_properties.filtered( + lambda x: x.company_id and + x.fields_id == prop.fields_id + ).ids, + ) + ]).ids + ), + ]) + + for redundant_property in self.env['ir.property'].search(domain): + result.append({ + 'name': '%s@%s: %s' % ( + prop.name, prop.res_id, prop.get_by_record(prop) + ), + 'property_id': redundant_property.id, + 'reason': REASON_DEFAULT, + }) + handled_field_ids.append(prop.fields_id.id) + self.env.cr.execute( + ''' + with grouped_properties(ids, cnt) as ( + select array_agg(id), count(*) + from ir_property group by res_id, company_id, fields_id + ) + select ids from grouped_properties where cnt > 1 + ''' + ) + for ids, in self.env.cr.fetchall(): + # odoo uses the first property found by search + for prop in self.env['ir.property'].search([ + ('id', 'in', ids) + ])[1:]: + result.append({ + 'name': '%s@%s: %s' % ( + prop.name, prop.res_id, prop.get_by_record(prop) + ), + 'property_id': prop.id, + 'reason': REASON_DUPLICATE, + }) + + return result + + purge_line_ids = fields.One2many( + 'cleanup.purge.line.property', 'wizard_id', 'Properties to purge') diff --git a/database_cleanup/tests/test_database_cleanup.py b/database_cleanup/tests/test_database_cleanup.py index f33b21ea3..a886210e6 100644 --- a/database_cleanup/tests/test_database_cleanup.py +++ b/database_cleanup/tests/test_database_cleanup.py @@ -17,6 +17,21 @@ class TestDatabaseCleanup(TransactionCase): self.model = None def test_database_cleanup(self): + # delete some index and check if our module recreated it + self.env.cr.execute('drop index res_partner_name_index') + create_indexes = self.env['cleanup.create_indexes.wizard'].create({}) + create_indexes.purge_all() + self.env.cr.execute( + 'select indexname from pg_indexes ' + "where indexname='res_partner_name_index' and " + "tablename='res_partner'" + ) + self.assertEqual(self.env.cr.rowcount, 1) + # duplicate a property + duplicate_property = self.env['ir.property'].search([], limit=1).copy() + purge_property = self.env['cleanup.purge.wizard.property'].create({}) + purge_property.purge_all() + self.assertFalse(duplicate_property.exists()) # create an orphaned column self.cr.execute( 'alter table res_partner add column database_cleanup_test int') diff --git a/database_cleanup/views/create_indexes.xml b/database_cleanup/views/create_indexes.xml new file mode 100644 index 000000000..b89c3993a --- /dev/null +++ b/database_cleanup/views/create_indexes.xml @@ -0,0 +1,52 @@ + + + + cleanup.create_indexes.wizard + + primary + + + + + + + + Create missing indexes + ir.actions.server + code + + action = self.get_wizard_action(cr, uid, context=context) + + + + cleanup.create_indexes.line + + primary + + + + + + + Create + ir.actions.server + code + + self.purge(cr, uid, context.get('active_ids', []), context) + + + + Create indexes + action + client_action_multi + cleanup.create_indexes.line + + + diff --git a/database_cleanup/views/menu.xml b/database_cleanup/views/menu.xml index 0796f907b..c669ddfb8 100644 --- a/database_cleanup/views/menu.xml +++ b/database_cleanup/views/menu.xml @@ -52,5 +52,19 @@ + + Create missing indexes + + + + + + + Purge obsolete properties + + + + + diff --git a/database_cleanup/views/purge_properties.xml b/database_cleanup/views/purge_properties.xml new file mode 100644 index 000000000..cf9b8c456 --- /dev/null +++ b/database_cleanup/views/purge_properties.xml @@ -0,0 +1,49 @@ + + + + + cleanup.purge.wizard.property + + primary + + + + + + + Purge properties + ir.actions.server + code + + action = self.get_wizard_action(cr, uid, context=context) + + + + cleanup.purge.line.property + + primary + + + + + + + + + Purge + ir.actions.server + code + + self.purge(cr, uid, context.get('active_ids', []), context) + + + + Purge + action + client_action_multi + cleanup.purge.line.property + + + + + diff --git a/database_cleanup/views/purge_wizard.xml b/database_cleanup/views/purge_wizard.xml index 40417f3a8..0fd88e793 100644 --- a/database_cleanup/views/purge_wizard.xml +++ b/database_cleanup/views/purge_wizard.xml @@ -9,7 +9,7 @@