From f08a8e10242ba17e02569d4644600758d6a3eed3 Mon Sep 17 00:00:00 2001 From: Stefan Rijnhart Date: Sat, 8 Feb 2014 18:00:09 +0100 Subject: [PATCH] [FIX] Don't remove uid field from wkf_instance, which is written in raw SQL query (but never read afterwards). Workaround for lp:1277899 [FIX] Preserve dangling workflow table which is in use [RFR] Group models per table when detecting columns to purge to prevent problems with models sharing the same table [ADD] Allow purging of dangling data entries [FIX] Data purging now working [IMP] Docstrings [FIX] Label [FIX] Catch attempt to unlink field from nonexisting model [RFR] Flake8 --- database_cleanup/__openerp__.py | 5 +- database_cleanup/model/__init__.py | 1 + database_cleanup/model/purge_columns.py | 43 +++++++--- database_cleanup/model/purge_data.py | 106 ++++++++++++++++++++++++ database_cleanup/model/purge_models.py | 19 +++-- database_cleanup/model/purge_modules.py | 2 +- database_cleanup/model/purge_tables.py | 17 ++-- database_cleanup/model/purge_wizard.py | 1 + database_cleanup/view/menu.xml | 9 +- database_cleanup/view/purge_data.xml | 37 +++++++++ 10 files changed, 211 insertions(+), 29 deletions(-) create mode 100644 database_cleanup/model/purge_data.py create mode 100644 database_cleanup/view/purge_data.xml diff --git a/database_cleanup/__openerp__.py b/database_cleanup/__openerp__.py index c752ae48e..12c723365 100644 --- a/database_cleanup/__openerp__.py +++ b/database_cleanup/__openerp__.py @@ -31,12 +31,13 @@ 'view/purge_models.xml', 'view/purge_columns.xml', 'view/purge_tables.xml', + 'view/purge_data.xml', 'view/menu.xml', ], 'description': """\ Clean your OpenERP database from remnants of modules, models, columns and -tables left by uninstalled modules (prior to 7.0) or a homebrew database upgrade -to a new major version of OpenERP. +tables left by uninstalled modules (prior to 7.0) or a homebrew database +upgrade to a new major version of OpenERP. After installation of this module, go to the Settings menu -> Technical -> Database cleanup. Go through the modules, models, columns and tables diff --git a/database_cleanup/model/__init__.py b/database_cleanup/model/__init__.py index 9b366b62b..77faf891c 100644 --- a/database_cleanup/model/__init__.py +++ b/database_cleanup/model/__init__.py @@ -3,3 +3,4 @@ from . import purge_modules from . import purge_models from . import purge_columns from . import purge_tables +from . import purge_data diff --git a/database_cleanup/model/purge_columns.py b/database_cleanup/model/purge_columns.py index 5266ab160..60b56ccc8 100644 --- a/database_cleanup/model/purge_columns.py +++ b/database_cleanup/model/purge_columns.py @@ -53,7 +53,7 @@ class CleanupPurgeLineColumn(orm.TransientModel): 'WHERE attrelid = ' '( SELECT oid FROM pg_class WHERE relname = %s ) ' 'AND attname = %s', - (model_pool._table, line.name)); + (model_pool._table, line.name)) if not cr.fetchone()[0]: continue @@ -68,10 +68,17 @@ class CleanupPurgeLineColumn(orm.TransientModel): cr.commit() return True + class CleanupPurgeWizardColumn(orm.TransientModel): _inherit = 'cleanup.purge.wizard' _name = 'cleanup.purge.wizard.column' + # List of known columns in use without corresponding fields + # Format: {table: [fields]} + blacklist = { + 'wkf_instance': ['uid'], # lp:1277899 + } + def default_get(self, cr, uid, fields, context=None): res = super(CleanupPurgeWizardColumn, self).default_get( cr, uid, fields, context=context) @@ -79,17 +86,22 @@ class CleanupPurgeWizardColumn(orm.TransientModel): res['name'] = _('Purge columns') return res - def get_orphaned_columns(self, cr, uid, model_pool, context=None): + def get_orphaned_columns(self, cr, uid, model_pools, context=None): """ From openobject-server/openerp/osv/orm.py Iterate on the database columns to identify columns of fields which have been removed """ - columns = [ - c for c in model_pool._columns - if not (isinstance(model_pool._columns[c], fields.function) - and not model_pool._columns[c].store)] + + columns = list(set([ + column for model_pool in model_pools + for column in model_pool._columns + if not (isinstance(model_pool._columns[column], fields.function) + and not model_pool._columns[column].store) + ])) columns += orm.MAGIC_COLUMNS + columns += self.blacklist.get(model_pools[0]._table, []) + cr.execute("SELECT a.attname" " FROM pg_class c, pg_attribute a" " WHERE c.relname=%s" @@ -98,26 +110,37 @@ class CleanupPurgeWizardColumn(orm.TransientModel): " AND pg_catalog.format_type(a.atttypid, a.atttypmod)" " NOT IN ('cid', 'tid', 'oid', 'xid')" " AND a.attname NOT IN %s", - (model_pool._table, False, tuple(columns))), + (model_pools[0]._table, False, tuple(columns))), return [column[0] for column in cr.fetchall()] def find(self, cr, uid, context=None): """ Search for columns that are not in the corresponding model. + + Group models by table to prevent false positives for columns + that are only in some of the models sharing the same table. + Example of this is 'sale_id' not being a field of stock.picking.in """ res = [] model_pool = self.pool['ir.model'] model_ids = model_pool.search(cr, uid, [], context=context) - line_pool = self.pool['cleanup.purge.line.column'] + + # mapping of tables to tuples (model id, [pool1, pool2, ...]) + table2model = {} + for model in model_pool.browse(cr, uid, model_ids, context=context): model_pool = self.pool.get(model.model) if not model_pool or not model_pool._auto: continue + table2model.setdefault( + model_pool._table, (model.id, []))[1].append(model_pool) + + for table, model_spec in table2model.iteritems(): for column in self.get_orphaned_columns( - cr, uid, model_pool, context=context): + cr, uid, model_spec[1], context=context): res.append((0, 0, { 'name': column, - 'model_id': model.id})) + 'model_id': model_spec[0]})) if not res: raise orm.except_orm( _('Nothing to do'), diff --git a/database_cleanup/model/purge_data.py b/database_cleanup/model/purge_data.py new file mode 100644 index 000000000..bcab2d832 --- /dev/null +++ b/database_cleanup/model/purge_data.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# This module copyright (C) 2014 Therp BV (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +from openerp.osv import orm, fields +from openerp.tools.translate import _ + + +class CleanupPurgeLineData(orm.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.data' + + _columns = { + 'data_id': fields.many2one( + 'ir.model.data', 'Data entry', + ondelete='SET NULL'), + 'wizard_id': fields.many2one( + 'cleanup.purge.wizard.data', 'Purge Wizard', readonly=True), + } + + def purge(self, cr, uid, ids, context=None): + """ + Unlink data entries upon manual confirmation. + """ + data_ids = [] + for line in self.browse(cr, uid, ids, context=context): + if line.purged or not line.data_id: + continue + data_ids.append(line.data_id.id) + self.logger.info('Purging data entry: %s', line.name) + self.pool['ir.model.data'].unlink(cr, uid, data_ids, context=context) + return self.write(cr, uid, ids, {'purged': True}, context=context) + + +class CleanupPurgeWizardData(orm.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.data' + + def default_get(self, cr, uid, fields, context=None): + res = super(CleanupPurgeWizardData, self).default_get( + cr, uid, fields, context=context) + if 'name' in fields: + res['name'] = _('Purge data') + return res + + def find(self, cr, uid, context=None): + """ + Collect all rows from ir_model_data that refer + to a nonexisting model, or to a nonexisting + row in the model's table. + """ + res = [] + data_pool = self.pool['ir.model.data'] + data_ids = [] + unknown_models = [] + cr.execute("""SELECT DISTINCT(model) FROM ir_model_data""") + for (model,) in cr.fetchall(): + if not model: + continue + if not self.pool.get(model): + unknown_models.append(model) + continue + cr.execute( + """ + SELECT id FROM ir_model_data + WHERE model = %%s + AND res_id IS NOT NULL + AND res_id NOT IN ( + SELECT id FROM %s) + """ % self.pool[model]._table, (model,)) + data_ids += [data_row[0] for data_row in cr.fetchall()] + data_ids += data_pool.search( + cr, uid, [('model', 'in', unknown_models)], context=context) + for data in data_pool.browse(cr, uid, data_ids, context=context): + res.append((0, 0, { + 'data_id': data.id, + 'name': "%s.%s, object of type %s" % ( + data.module, data.name, data.model)})) + if not res: + raise orm.except_orm( + _('Nothing to do'), + _('No orphaned data entries found')) + return res + + _columns = { + 'purge_line_ids': fields.one2many( + 'cleanup.purge.line.data', + 'wizard_id', 'Data to purge'), + } diff --git a/database_cleanup/model/purge_models.py b/database_cleanup/model/purge_models.py index 811da30fe..a6342afd3 100644 --- a/database_cleanup/model/purge_models.py +++ b/database_cleanup/model/purge_models.py @@ -53,11 +53,11 @@ class CleanupPurgeLineModel(orm.TransientModel): constraint_pool = self.pool['ir.model.constraint'] fields_pool = self.pool['ir.model.fields'] - local_context=(context or {}).copy() + local_context = (context or {}).copy() local_context.update({ - MODULE_UNINSTALL_FLAG: True, - 'no_drop_table': True, - }) + MODULE_UNINSTALL_FLAG: True, + 'no_drop_table': True, + }) for line in self.browse(cr, uid, ids, context=context): cr.execute( @@ -80,9 +80,14 @@ class CleanupPurgeLineModel(orm.TransientModel): cr, uid, constraint_ids, context=context) relation_ids = fields_pool.search( cr, uid, [('relation', '=', row[1])], context=context) - if relation_ids: - fields_pool.unlink(cr, uid, relation_ids, - context=local_context) + for relation in relation_ids: + try: + # Fails if the model on the target side + # cannot be instantiated + fields_pool.unlink(cr, uid, [relation], + context=local_context) + except AttributeError: + pass model_pool.unlink(cr, uid, [row[0]], context=local_context) line.write({'purged': True}) cr.commit() diff --git a/database_cleanup/model/purge_modules.py b/database_cleanup/model/purge_modules.py index b62a037a4..12734d157 100644 --- a/database_cleanup/model/purge_modules.py +++ b/database_cleanup/model/purge_modules.py @@ -50,7 +50,7 @@ class CleanupPurgeLineModule(orm.TransientModel): module_pool.write( cr, uid, module_ids, {'state': 'to remove'}, context=context) cr.commit() - _db, _pool = pooler.restart_pool(cr.dbname, update_module=True) + _db, _pool = pooler.restart_pool(cr.dbname, update_module=True) module_pool.unlink(cr, uid, module_ids, context=context) return self.write(cr, uid, ids, {'purged': True}, context=context) diff --git a/database_cleanup/model/purge_tables.py b/database_cleanup/model/purge_tables.py index 8fb6120e2..8d9c86544 100644 --- a/database_cleanup/model/purge_tables.py +++ b/database_cleanup/model/purge_tables.py @@ -56,11 +56,11 @@ class CleanupPurgeLineTable(orm.TransientModel): FROM pg_attribute af, pg_attribute a, (SELECT conname, conrelid, confrelid,conkey[i] AS conkey, confkey[i] AS confkey - FROM (select conname, conrelid, confrelid, conkey, confkey, - generate_series(1,array_upper(conkey,1)) AS i + FROM (select conname, conrelid, confrelid, conkey, + confkey, generate_series(1,array_upper(conkey,1)) AS i FROM pg_constraint WHERE contype = 'f') ss) ss2 WHERE af.attnum = confkey AND af.attrelid = confrelid AND - a.attnum = conkey AND a.attrelid = conrelid + a.attnum = conkey AND a.attrelid = conrelid AND confrelid::regclass = '%s'::regclass; """ % line.name) @@ -80,6 +80,7 @@ class CleanupPurgeLineTable(orm.TransientModel): cr.commit() return True + class CleanupPurgeWizardTable(orm.TransientModel): _inherit = 'cleanup.purge.wizard' _name = 'cleanup.purge.wizard.table' @@ -88,7 +89,7 @@ class CleanupPurgeWizardTable(orm.TransientModel): res = super(CleanupPurgeWizardTable, self).default_get( cr, uid, fields, context=context) if 'name' in fields: - res['name'] = _('Purge modules') + res['name'] = _('Purge tables') return res def find(self, cr, uid, context=None): @@ -97,11 +98,11 @@ class CleanupPurgeWizardTable(orm.TransientModel): Ignore views for now. """ model_ids = self.pool['ir.model'].search(cr, uid, [], context=context) - line_pool = self.pool['cleanup.purge.line.table'] - known_tables = [] + # Start out with known tables with no model + known_tables = ['wkf_witm_trans'] for model in self.pool['ir.model'].browse( cr, uid, model_ids, context=context): - + model_pool = self.pool.get(model.model) if not model_pool: continue @@ -119,7 +120,7 @@ class CleanupPurgeWizardTable(orm.TransientModel): [("'%s'" % table) for table in known_tables]) cr.execute( """ - SELECT table_name FROM information_schema.tables + SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE' AND table_name NOT IN (%s)""" % known_tables_repr) diff --git a/database_cleanup/model/purge_wizard.py b/database_cleanup/model/purge_wizard.py index 542ac1507..5003c4aef 100644 --- a/database_cleanup/model/purge_wizard.py +++ b/database_cleanup/model/purge_wizard.py @@ -36,6 +36,7 @@ class CleanupPurgeLine(orm.AbstractModel): def purge(self, cr, uid, ids, context=None): raise NotImplementedError + class PurgeWizard(orm.AbstractModel): """ Abstract base class for the purge wizards """ _name = 'cleanup.purge.wizard' diff --git a/database_cleanup/view/menu.xml b/database_cleanup/view/menu.xml index ff6a1694b..9d770ea03 100644 --- a/database_cleanup/view/menu.xml +++ b/database_cleanup/view/menu.xml @@ -32,10 +32,17 @@ Purge obsolete tables - + + + Purge obsolete data entries + + + + + diff --git a/database_cleanup/view/purge_data.xml b/database_cleanup/view/purge_data.xml new file mode 100644 index 000000000..e749f4569 --- /dev/null +++ b/database_cleanup/view/purge_data.xml @@ -0,0 +1,37 @@ + + + + + + Form view for purge data wizard + cleanup.purge.wizard.data + +
+

+ +

+