commit f3eb39e15b629caf1895725c7041ec05242b8212 Author: Stefan Rijnhart Date: Tue Jan 28 22:09:41 2014 +0100 [ADD] Database cleanup module diff --git a/database_cleanup/__init__.py b/database_cleanup/__init__.py new file mode 100644 index 000000000..9186ee3ad --- /dev/null +++ b/database_cleanup/__init__.py @@ -0,0 +1 @@ +from . import model diff --git a/database_cleanup/__openerp__.py b/database_cleanup/__openerp__.py new file mode 100644 index 000000000..c752ae48e --- /dev/null +++ b/database_cleanup/__openerp__.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# This module copyright (C) 2014 Therp BV (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +{ + 'name': 'Database cleanup', + 'version': '0.1', + 'author': 'Therp BV', + 'depends': ['base'], + 'license': 'AGPL-3', + 'category': 'Tools', + 'data': [ + 'view/purge_modules.xml', + 'view/purge_models.xml', + 'view/purge_columns.xml', + 'view/purge_tables.xml', + 'view/menu.xml', + ], + 'description': """\ +Clean your OpenERP database from remnants of modules, models, columns and +tables left by uninstalled modules (prior to 7.0) or a homebrew database upgrade +to a new major version of OpenERP. + +After installation of this module, go to the Settings menu -> Technical -> +Database cleanup. Go through the modules, models, columns and tables +entries under this menu (in that order) and find out if there is orphaned data +in your database. You can either delete entries by line, or sweep all entries +in one big step (if you are *really* confident). + +Caution! This module is potentially harmful and can *easily* destroy the +integrity of your data. Do not use if you are not entirely comfortable +with the technical details of the OpenERP data model of *all* the modules +that have ever been installed on your database, and do not purge any module, +model, column or table if you do not know exactly what you are doing. +""", + +} diff --git a/database_cleanup/model/__init__.py b/database_cleanup/model/__init__.py new file mode 100644 index 000000000..9b366b62b --- /dev/null +++ b/database_cleanup/model/__init__.py @@ -0,0 +1,5 @@ +from . import purge_wizard +from . import purge_modules +from . import purge_models +from . import purge_columns +from . import purge_tables diff --git a/database_cleanup/model/purge_columns.py b/database_cleanup/model/purge_columns.py new file mode 100644 index 000000000..6efc3094f --- /dev/null +++ b/database_cleanup/model/purge_columns.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# This module copyright (C) 2014 Therp BV (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +from openerp.osv import orm, fields +from openerp.tools.translate import _ + + +class CleanupPurgeLineColumn(orm.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.column' + + _columns = { + 'model_id': fields.many2one( + 'ir.model', 'Model', + required=True, ondelete='CASCADE'), + 'wizard_id': fields.many2one( + 'cleanup.purge.wizard.column', 'Purge Wizard', readonly=True), + } + + def purge(self, cr, uid, ids, context=None): + """ + Unlink columns upon manual confirmation. + """ + for line in self.browse(cr, uid, ids, context=context): + if line.purged: + continue + + model_pool = self.pool[line.model_id.model] + + # Check whether the column actually still exists. + # Inheritance such as stock.picking.in from stock.picking + # can lead to double attempts at removal + cr.execute( + 'SELECT count(attname) FROM pg_attribute ' + 'WHERE attrelid = ' + '( SELECT oid FROM pg_class WHERE relname = %s ) ' + 'AND attname = %s', + (model_pool._table, line.name)); + if not cr.fetchone()[0]: + continue + + self.logger.info( + 'Dropping column %s from table %s', + line.name, model_pool._table) + cr.execute( + """ + ALTER TABLE "%s" DROP COLUMN "%s" + """ % (model_pool._table, line.name)) + line.write({'purged': True}) + cr.commit() + return True + +class CleanupPurgeWizardColumn(orm.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.column' + + def default_get(self, cr, uid, fields, context=None): + res = super(CleanupPurgeWizardColumn, self).default_get( + cr, uid, fields, context=context) + if 'name' in fields: + res['name'] = _('Purge columns') + return res + + def get_orphaned_columns(self, cr, uid, model_pool, context=None): + """ + From openobject-server/openerp/osv/orm.py + Iterate on the database columns to identify columns + of fields which have been removed + """ + columns = [ + c for c in model_pool._columns + if not (isinstance(model_pool._columns[c], fields.function) + and not model_pool._columns[c].store)] + columns += orm.MAGIC_COLUMNS + cr.execute("SELECT a.attname" + " FROM pg_class c, pg_attribute a" + " WHERE c.relname=%s" + " AND c.oid=a.attrelid" + " AND a.attisdropped=%s" + " AND pg_catalog.format_type(a.atttypid, a.atttypmod)" + " NOT IN ('cid', 'tid', 'oid', 'xid')" + " AND a.attname NOT IN %s", + (model_pool._table, False, tuple(columns))), + return [column[0] for column in cr.fetchall()] + + def find(self, cr, uid, context=None): + """ + Search for columns that cannot be instanciated. + """ + res = [] + model_pool = self.pool['ir.model'] + model_ids = model_pool.search(cr, uid, [], context=context) + line_pool = self.pool['cleanup.purge.line.column'] + for model in model_pool.browse(cr, uid, model_ids, context=context): + model_pool = self.pool.get(model.model) + if not model_pool or not model_pool._auto: + continue + for column in self.get_orphaned_columns( + cr, uid, model_pool, context=context): + res.append((0, 0, { + 'name': column, + 'model_id': model.id})) + if not res: + raise orm.except_orm( + _('Nothing to do'), + _('No orphaned columns found')) + return res + + _columns = { + 'purge_line_ids': fields.one2many( + 'cleanup.purge.line.column', + 'wizard_id', 'Columns to purge'), + } diff --git a/database_cleanup/model/purge_models.py b/database_cleanup/model/purge_models.py new file mode 100644 index 000000000..f2a1dda09 --- /dev/null +++ b/database_cleanup/model/purge_models.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +import logging +from openerp.osv import orm, fields +from openerp.tools.translate import _ +from openerp.addons.base.ir.ir_model import MODULE_UNINSTALL_FLAG + + +class IrModel(orm.Model): + _inherit = 'ir.model' + + def _drop_table(self, cr, uid, ids, context=None): + # Allow to skip this step during model unlink + # The super method crashes if the model cannot be instantiated + if context and context.get('no_drop_table'): + return True + return super(IrModel, self)._drop_table(cr, uid, ids, context=context) + + +class CleanupPurgeLineModel(orm.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.model' + + _columns = { + 'wizard_id': fields.many2one( + 'cleanup.purge.wizard.model', 'Purge Wizard', readonly=True), + } + + def purge(self, cr, uid, ids, context=None): + """ + Unlink models upon manual confirmation. + """ + model_pool = self.pool['ir.model'] + attachment_pool = self.pool['ir.attachment'] + constraint_pool = self.pool['ir.model.constraint'] + + local_context=(context or {}).copy() + local_context.update({ + MODULE_UNINSTALL_FLAG: True, + 'no_drop_table': True, + }) + + for line in self.browse(cr, uid, ids, context=context): + cr.execute( + "SELECT id, model from ir_model WHERE model = %s", + (line.name,)) + row = cr.fetchone() + if row: + self.logger.info('Purging model %s', row[1]) + attachment_ids = attachment_pool.search( + cr, uid, [('res_model', '=', line.name)], context=context) + if attachment_ids: + attachment_pool.write( + cr, uid, attachment_ids, {'res_model': False}, + context=context) + constraint_ids = constraint_pool.search( + cr, uid, [('model', '=', line.name)], context=context) + if constraint_ids: + constraint_pool.unlink( + cr, uid, constraint_ids, context=context) + model_pool.unlink(cr, uid, [row[0]], context=local_context) + line.write({'purged': True}) + cr.commit() + return True + + +class CleanupPurgeWizardModel(orm.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.model' + + def default_get(self, cr, uid, fields, context=None): + res = super(CleanupPurgeWizardModel, self).default_get( + cr, uid, fields, context=context) + if 'name' in fields: + res['name'] = _('Purge models') + return res + + def find(self, cr, uid, context=None): + """ + Search for models that cannot be instanciated. + """ + res = [] + cr.execute("SELECT model from ir_model") + for (model,) in cr.fetchall(): + if not self.pool.get(model): + res.append((0, 0, {'name': model})) + if not res: + raise orm.except_orm( + _('Nothing to do'), + _('No orphaned models found')) + return res + + _columns = { + 'purge_line_ids': fields.one2many( + 'cleanup.purge.line.model', + 'wizard_id', 'Models to purge'), + } diff --git a/database_cleanup/model/purge_modules.py b/database_cleanup/model/purge_modules.py new file mode 100644 index 000000000..b62a037a4 --- /dev/null +++ b/database_cleanup/model/purge_modules.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# This module copyright (C) 2014 Therp BV (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +from openerp import pooler +from openerp.osv import orm, fields +from openerp.modules.module import get_module_path +from openerp.tools.translate import _ + + +class CleanupPurgeLineModule(orm.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.module' + + _columns = { + 'wizard_id': fields.many2one( + 'cleanup.purge.wizard.module', 'Purge Wizard', readonly=True), + } + + def purge(self, cr, uid, ids, context=None): + """ + Uninstall modules upon manual confirmation, then reload + the database. + """ + module_pool = self.pool['ir.module.module'] + lines = self.browse(cr, uid, ids, context=context) + module_names = [line.name for line in lines if not line.purged] + module_ids = module_pool.search( + cr, uid, [('name', 'in', module_names)], context=context) + if not module_ids: + return True + self.logger.info('Purging modules %s', ', '.join(module_names)) + module_pool.write( + cr, uid, module_ids, {'state': 'to remove'}, context=context) + cr.commit() + _db, _pool = pooler.restart_pool(cr.dbname, update_module=True) + module_pool.unlink(cr, uid, module_ids, context=context) + return self.write(cr, uid, ids, {'purged': True}, context=context) + + +class CleanupPurgeWizardModule(orm.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.module' + + def default_get(self, cr, uid, fields, context=None): + res = super(CleanupPurgeWizardModule, self).default_get( + cr, uid, fields, context=context) + if 'name' in fields: + res['name'] = _('Purge modules') + return res + + def find(self, cr, uid, context=None): + module_pool = self.pool['ir.module.module'] + module_ids = module_pool.search(cr, uid, [], context=context) + res = [] + for module in module_pool.browse(cr, uid, module_ids, context=context): + if get_module_path(module.name): + continue + if module.state == 'uninstalled': + module_pool.unlink(cr, uid, module.id, context=context) + continue + res.append((0, 0, {'name': module.name})) + + if not res: + raise orm.except_orm( + _('Nothing to do'), + _('No modules found to purge')) + return res + + _columns = { + 'purge_line_ids': fields.one2many( + 'cleanup.purge.line.module', + 'wizard_id', 'Modules to purge'), + } diff --git a/database_cleanup/model/purge_tables.py b/database_cleanup/model/purge_tables.py new file mode 100644 index 000000000..8fb6120e2 --- /dev/null +++ b/database_cleanup/model/purge_tables.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# This module copyright (C) 2014 Therp BV (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +from openerp.osv import orm, fields +from openerp.tools.translate import _ + + +class CleanupPurgeLineTable(orm.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.table' + + _columns = { + 'wizard_id': fields.many2one( + 'cleanup.purge.wizard.table', 'Purge Wizard', readonly=True), + } + + def purge(self, cr, uid, ids, context=None): + """ + Unlink tables upon manual confirmation. + """ + lines = self.browse(cr, uid, ids, context=context) + tables = [line.name for line in lines] + for line in lines: + if line.purged: + continue + + # Retrieve constraints on the tables to be dropped + # This query is referenced in numerous places + # on the Internet but credits probably go to Tom Lane + # in this post http://www.postgresql.org/\ + # message-id/22895.1226088573@sss.pgh.pa.us + # Only using the constraint name and the source table, + # but I'm leaving the rest in for easier debugging + cr.execute( + """ + SELECT conname, confrelid::regclass, af.attname AS fcol, + conrelid::regclass, a.attname AS col + FROM pg_attribute af, pg_attribute a, + (SELECT conname, conrelid, confrelid,conkey[i] AS conkey, + confkey[i] AS confkey + FROM (select conname, conrelid, confrelid, conkey, confkey, + generate_series(1,array_upper(conkey,1)) AS i + FROM pg_constraint WHERE contype = 'f') ss) ss2 + WHERE af.attnum = confkey AND af.attrelid = confrelid AND + a.attnum = conkey AND a.attrelid = conrelid + AND confrelid::regclass = '%s'::regclass; + """ % line.name) + + for constraint in cr.fetchall(): + if constraint[3] in tables: + self.logger.info( + 'Dropping constraint %s on table %s (to be dropped)', + constraint[0], constraint[3]) + cr.execute( + "ALTER TABLE %s DROP CONSTRAINT %s" % ( + constraint[3], constraint[0])) + + self.logger.info( + 'Dropping table %s', line.name) + cr.execute("DROP TABLE \"%s\"" % (line.name,)) + line.write({'purged': True}) + cr.commit() + return True + +class CleanupPurgeWizardTable(orm.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.table' + + def default_get(self, cr, uid, fields, context=None): + res = super(CleanupPurgeWizardTable, self).default_get( + cr, uid, fields, context=context) + if 'name' in fields: + res['name'] = _('Purge modules') + return res + + def find(self, cr, uid, context=None): + """ + Search for tables that cannot be instantiated. + Ignore views for now. + """ + model_ids = self.pool['ir.model'].search(cr, uid, [], context=context) + line_pool = self.pool['cleanup.purge.line.table'] + known_tables = [] + for model in self.pool['ir.model'].browse( + cr, uid, model_ids, context=context): + + model_pool = self.pool.get(model.model) + if not model_pool: + continue + known_tables.append(model_pool._table) + known_tables += [ + column._sql_names(model_pool)[0] + for column in model_pool._columns.values() + if column._type == 'many2many' + # unstored function fields of type m2m don't have _rel + and hasattr(column, '_rel') + ] + + # Cannot pass table names as a psycopg argument + known_tables_repr = ",".join( + [("'%s'" % table) for table in known_tables]) + cr.execute( + """ + SELECT table_name FROM information_schema.tables + WHERE table_schema = 'public' AND table_type = 'BASE TABLE' + AND table_name NOT IN (%s)""" % known_tables_repr) + + res = [(0, 0, {'name': row[0]}) for row in cr.fetchall()] + if not res: + raise orm.except_orm( + _('Nothing to do'), + _('No orphaned tables found')) + return res + + _columns = { + 'purge_line_ids': fields.one2many( + 'cleanup.purge.line.table', + 'wizard_id', 'Tables to purge'), + } diff --git a/database_cleanup/model/purge_wizard.py b/database_cleanup/model/purge_wizard.py new file mode 100644 index 000000000..542ac1507 --- /dev/null +++ b/database_cleanup/model/purge_wizard.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# This module copyright (C) 2014 Therp BV (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +import logging +from openerp.osv import orm, fields + + +class CleanupPurgeLine(orm.AbstractModel): + """ Abstract base class for the purge wizard lines """ + _name = 'cleanup.purge.line' + _columns = { + 'name': fields.char('Name', size=256, readonly=True), + 'purged': fields.boolean('Purged', readonly=True), + } + + logger = logging.getLogger('openerp.addons.database_cleanup') + + def purge(self, cr, uid, ids, context=None): + raise NotImplementedError + +class PurgeWizard(orm.AbstractModel): + """ Abstract base class for the purge wizards """ + _name = 'cleanup.purge.wizard' + + def default_get(self, cr, uid, fields, context=None): + res = super(PurgeWizard, self).default_get( + cr, uid, fields, context=context) + if 'purge_line_ids' in fields: + res['purge_line_ids'] = self.find(cr, uid, context=None) + return res + + def find(self, cr, uid, ids, context=None): + raise NotImplementedError + + def purge_all(self, cr, uid, ids, context=None): + line_pool = self.pool[self._columns['purge_line_ids']._obj] + for wizard in self.browse(cr, uid, ids, context=context): + line_pool.purge( + cr, uid, [line.id for line in wizard.purge_line_ids], + context=context) + return True + + _columns = { + 'name': fields.char('Name', size=64, readonly=True), + } diff --git a/database_cleanup/static/src/img/icon.png b/database_cleanup/static/src/img/icon.png new file mode 100644 index 000000000..6980d05de Binary files /dev/null and b/database_cleanup/static/src/img/icon.png differ diff --git a/database_cleanup/view/menu.xml b/database_cleanup/view/menu.xml new file mode 100644 index 000000000..ff6a1694b --- /dev/null +++ b/database_cleanup/view/menu.xml @@ -0,0 +1,41 @@ + + + + + + Database cleanup + + + + + + + Purge obsolete modules + + + + + + + Purge obsolete models + + + + + + + Purge obsolete columns + + + + + + + Purge obsolete tables + + + + + + + diff --git a/database_cleanup/view/purge_columns.xml b/database_cleanup/view/purge_columns.xml new file mode 100644 index 000000000..40ed4a4f6 --- /dev/null +++ b/database_cleanup/view/purge_columns.xml @@ -0,0 +1,37 @@ + + + + + + Form view for purge columns wizard + cleanup.purge.wizard.column + +
+

+ +

+