From 79bf9fd8f66043cf540ebd8c05dcbdb19b03291a Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Wed, 29 Jun 2016 16:37:51 +0200 Subject: [PATCH] [ADD] migrate database_cleanup [ADD] test purging modules [ADD] test purging tables --- database_cleanup/README.rst | 63 ++++++- database_cleanup/__init__.py | 5 +- database_cleanup/__openerp__.py | 43 ++--- database_cleanup/identifier_adapter.py | 23 +++ database_cleanup/model/purge_columns.py | 155 ------------------ database_cleanup/model/purge_data.py | 106 ------------ database_cleanup/model/purge_menus.py | 81 --------- database_cleanup/model/purge_models.py | 155 ------------------ database_cleanup/model/purge_modules.py | 129 --------------- database_cleanup/model/purge_tables.py | 138 ---------------- database_cleanup/model/purge_wizard.py | 87 ---------- .../{model => models}/__init__.py | 0 database_cleanup/models/purge_columns.py | 127 ++++++++++++++ database_cleanup/models/purge_data.py | 68 ++++++++ database_cleanup/models/purge_menus.py | 48 ++++++ database_cleanup/models/purge_models.py | 119 ++++++++++++++ database_cleanup/models/purge_modules.py | 81 +++++++++ database_cleanup/models/purge_tables.py | 106 ++++++++++++ database_cleanup/models/purge_wizard.py | 94 +++++++++++ database_cleanup/tests/__init__.py | 4 + .../tests/test_database_cleanup.py | 74 +++++++++ database_cleanup/{view => views}/menu.xml | 0 .../{view => views}/purge_columns.xml | 39 +---- .../{view => views}/purge_data.xml | 39 +---- .../{view => views}/purge_menus.xml | 34 +--- .../{view => views}/purge_models.xml | 35 +--- .../{view => views}/purge_modules.xml | 35 +--- .../{view => views}/purge_tables.xml | 31 +--- database_cleanup/views/purge_wizard.xml | 41 +++++ 29 files changed, 902 insertions(+), 1058 deletions(-) create mode 100644 database_cleanup/identifier_adapter.py delete mode 100644 database_cleanup/model/purge_columns.py delete mode 100644 database_cleanup/model/purge_data.py delete mode 100644 database_cleanup/model/purge_menus.py delete mode 100644 database_cleanup/model/purge_models.py delete mode 100644 database_cleanup/model/purge_modules.py delete mode 100644 database_cleanup/model/purge_tables.py delete mode 100644 database_cleanup/model/purge_wizard.py rename database_cleanup/{model => models}/__init__.py (100%) create mode 100644 database_cleanup/models/purge_columns.py create mode 100644 database_cleanup/models/purge_data.py create mode 100644 database_cleanup/models/purge_menus.py create mode 100644 database_cleanup/models/purge_models.py create mode 100644 database_cleanup/models/purge_modules.py create mode 100644 database_cleanup/models/purge_tables.py create mode 100644 database_cleanup/models/purge_wizard.py create mode 100644 database_cleanup/tests/__init__.py create mode 100644 database_cleanup/tests/test_database_cleanup.py rename database_cleanup/{view => views}/menu.xml (100%) rename database_cleanup/{view => views}/purge_columns.xml (54%) rename database_cleanup/{view => views}/purge_data.xml (53%) rename database_cleanup/{view => views}/purge_menus.xml (56%) rename database_cleanup/{view => views}/purge_models.xml (54%) rename database_cleanup/{view => views}/purge_modules.xml (54%) rename database_cleanup/{view => views}/purge_tables.xml (57%) create mode 100644 database_cleanup/views/purge_wizard.xml diff --git a/database_cleanup/README.rst b/database_cleanup/README.rst index 6456ff195..2adbf2b4a 100644 --- a/database_cleanup/README.rst +++ b/database_cleanup/README.rst @@ -1,15 +1,68 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 + +================ +Database cleanup +================ + Clean your OpenERP database from remnants of modules, models, columns and tables left by uninstalled modules (prior to 7.0) or a homebrew database upgrade to a new major version of OpenERP. +Caution! This module is potentially harmful and can *easily* destroy the +integrity of your data. Do not use if you are not entirely comfortable +with the technical details of the OpenERP data model of *all* the modules +that have ever been installed on your database, and do not purge any module, +model, column or table if you do not know exactly what you are doing. + +Usage +===== + After installation of this module, go to the Settings menu -> Technical -> Database cleanup. Go through the modules, models, columns and tables entries under this menu (in that order) and find out if there is orphaned data in your database. You can either delete entries by line, or sweep all entries in one big step (if you are *really* confident). -Caution! This module is potentially harmful and can *easily* destroy the -integrity of your data. Do not use if you are not entirely comfortable -with the technical details of the OpenERP data model of *all* the modules -that have ever been installed on your database, and do not purge any module, -model, column or table if you do not know exactly what you are doing. +.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas + :alt: Try me on Runbot + :target: https://runbot.odoo-community.org/runbot/149/9.0 + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed feedback. + +Credits +======= + +Images +------ + +* Odoo Community Association: `Icon `_. + +Contributors +------------ + +* Stefan Rijnhart +* Holger Brunn + +Do not contact contributors directly about help with questions or problems concerning this addon, but use the `community mailing list `_ or the `appropriate specialized mailinglist `_ for help, and the bug tracker linked in `Bug Tracker`_ above for technical issues. + +Maintainer +---------- + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit https://odoo-community.org. diff --git a/database_cleanup/__init__.py b/database_cleanup/__init__.py index 9186ee3ad..851a78463 100644 --- a/database_cleanup/__init__.py +++ b/database_cleanup/__init__.py @@ -1 +1,4 @@ -from . import model +# -*- coding: utf-8 -*- +# © 2014-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from . import models diff --git a/database_cleanup/__openerp__.py b/database_cleanup/__openerp__.py index a8abf1bf8..cf56734d3 100644 --- a/database_cleanup/__openerp__.py +++ b/database_cleanup/__openerp__.py @@ -1,39 +1,22 @@ # -*- coding: utf-8 -*- -############################################################################## -# -# OpenERP, Open Source Management Solution -# This module copyright (C) 2014 Therp BV (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - +# © 2014-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). { 'name': 'Database cleanup', - 'version': '8.0.0.1.0', + 'version': '9.0.1.0.0', 'author': "Therp BV,Odoo Community Association (OCA)", 'depends': ['base'], 'license': 'AGPL-3', 'category': 'Tools', 'data': [ - 'view/purge_menus.xml', - 'view/purge_modules.xml', - 'view/purge_models.xml', - 'view/purge_columns.xml', - 'view/purge_tables.xml', - 'view/purge_data.xml', - 'view/menu.xml', - ], - 'installable': False, + "views/purge_wizard.xml", + 'views/purge_menus.xml', + 'views/purge_modules.xml', + 'views/purge_models.xml', + 'views/purge_columns.xml', + 'views/purge_tables.xml', + 'views/purge_data.xml', + 'views/menu.xml', + ], + 'installable': True, } diff --git a/database_cleanup/identifier_adapter.py b/database_cleanup/identifier_adapter.py new file mode 100644 index 000000000..280fcd920 --- /dev/null +++ b/database_cleanup/identifier_adapter.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# © 2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from psycopg2.extensions import ISQLQuote + + +class IdentifierAdapter(ISQLQuote): + def __init__(self, identifier, quote=True): + self.quote = quote + self.identifier = identifier + + def __conform__(self, protocol): + if protocol == ISQLQuote: + return self + + def getquoted(self): + def is_identifier_char(c): + return c.isalnum() or c in ['_', '$'] + + format_string = '"%s"' + if not self.quote: + format_string = '%s' + return format_string % filter(is_identifier_char, self.identifier) diff --git a/database_cleanup/model/purge_columns.py b/database_cleanup/model/purge_columns.py deleted file mode 100644 index abcf1d2d4..000000000 --- a/database_cleanup/model/purge_columns.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# OpenERP, Open Source Management Solution -# This module copyright (C) 2014 Therp BV (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - -from openerp.osv import orm, fields -from openerp.tools.translate import _ - - -class CleanupPurgeLineColumn(orm.TransientModel): - _inherit = 'cleanup.purge.line' - _name = 'cleanup.purge.line.column' - - _columns = { - 'model_id': fields.many2one( - 'ir.model', 'Model', - required=True, ondelete='CASCADE'), - 'wizard_id': fields.many2one( - 'cleanup.purge.wizard.column', 'Purge Wizard', readonly=True), - } - - def purge(self, cr, uid, ids, context=None): - """ - Unlink columns upon manual confirmation. - """ - for line in self.browse(cr, uid, ids, context=context): - if line.purged: - continue - - model_pool = self.pool[line.model_id.model] - - # Check whether the column actually still exists. - # Inheritance such as stock.picking.in from stock.picking - # can lead to double attempts at removal - cr.execute( - 'SELECT count(attname) FROM pg_attribute ' - 'WHERE attrelid = ' - '( SELECT oid FROM pg_class WHERE relname = %s ) ' - 'AND attname = %s', - (model_pool._table, line.name)) - if not cr.fetchone()[0]: - continue - - self.logger.info( - 'Dropping column %s from table %s', - line.name, model_pool._table) - cr.execute( - """ - ALTER TABLE "%s" DROP COLUMN "%s" - """ % (model_pool._table, line.name)) - line.write({'purged': True}) - cr.commit() - return True - - -class CleanupPurgeWizardColumn(orm.TransientModel): - _inherit = 'cleanup.purge.wizard' - _name = 'cleanup.purge.wizard.column' - - # List of known columns in use without corresponding fields - # Format: {table: [fields]} - blacklist = { - 'wkf_instance': ['uid'], # lp:1277899 - } - - def default_get(self, cr, uid, fields, context=None): - res = super(CleanupPurgeWizardColumn, self).default_get( - cr, uid, fields, context=context) - if 'name' in fields: - res['name'] = _('Purge columns') - return res - - def get_orphaned_columns(self, cr, uid, model_pools, context=None): - """ - From openobject-server/openerp/osv/orm.py - Iterate on the database columns to identify columns - of fields which have been removed - """ - - columns = list(set([ - column for model_pool in model_pools - for column in model_pool._columns - if not (isinstance(model_pool._columns[column], - fields.function) and - not model_pool._columns[column].store) - ])) - columns += orm.MAGIC_COLUMNS - columns += self.blacklist.get(model_pools[0]._table, []) - - cr.execute("SELECT a.attname" - " FROM pg_class c, pg_attribute a" - " WHERE c.relname=%s" - " AND c.oid=a.attrelid" - " AND a.attisdropped=%s" - " AND pg_catalog.format_type(a.atttypid, a.atttypmod)" - " NOT IN ('cid', 'tid', 'oid', 'xid')" - " AND a.attname NOT IN %s", - (model_pools[0]._table, False, tuple(columns))), - return [column[0] for column in cr.fetchall()] - - def find(self, cr, uid, context=None): - """ - Search for columns that are not in the corresponding model. - - Group models by table to prevent false positives for columns - that are only in some of the models sharing the same table. - Example of this is 'sale_id' not being a field of stock.picking.in - """ - res = [] - model_pool = self.pool['ir.model'] - model_ids = model_pool.search(cr, uid, [], context=context) - - # mapping of tables to tuples (model id, [pool1, pool2, ...]) - table2model = {} - - for model in model_pool.browse(cr, uid, model_ids, context=context): - model_pool = self.pool.get(model.model) - if not model_pool or not model_pool._auto: - continue - table2model.setdefault( - model_pool._table, (model.id, []))[1].append(model_pool) - - for table, model_spec in table2model.iteritems(): - for column in self.get_orphaned_columns( - cr, uid, model_spec[1], context=context): - res.append((0, 0, { - 'name': column, - 'model_id': model_spec[0]})) - if not res: - raise orm.except_orm( - _('Nothing to do'), - _('No orphaned columns found')) - return res - - _columns = { - 'purge_line_ids': fields.one2many( - 'cleanup.purge.line.column', - 'wizard_id', 'Columns to purge'), - } diff --git a/database_cleanup/model/purge_data.py b/database_cleanup/model/purge_data.py deleted file mode 100644 index f3e1b63cd..000000000 --- a/database_cleanup/model/purge_data.py +++ /dev/null @@ -1,106 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# OpenERP, Open Source Management Solution -# This module copyright (C) 2014 Therp BV (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - -from openerp.osv import orm, fields -from openerp.tools.translate import _ - - -class CleanupPurgeLineData(orm.TransientModel): - _inherit = 'cleanup.purge.line' - _name = 'cleanup.purge.line.data' - - _columns = { - 'data_id': fields.many2one( - 'ir.model.data', 'Data entry', - ondelete='SET NULL'), - 'wizard_id': fields.many2one( - 'cleanup.purge.wizard.data', 'Purge Wizard', readonly=True), - } - - def purge(self, cr, uid, ids, context=None): - """ - Unlink data entries upon manual confirmation. - """ - data_ids = [] - for line in self.browse(cr, uid, ids, context=context): - if line.purged or not line.data_id: - continue - data_ids.append(line.data_id.id) - self.logger.info('Purging data entry: %s', line.name) - self.pool['ir.model.data'].unlink(cr, uid, data_ids, context=context) - return self.write(cr, uid, ids, {'purged': True}, context=context) - - -class CleanupPurgeWizardData(orm.TransientModel): - _inherit = 'cleanup.purge.wizard' - _name = 'cleanup.purge.wizard.data' - - def default_get(self, cr, uid, fields, context=None): - res = super(CleanupPurgeWizardData, self).default_get( - cr, uid, fields, context=context) - if 'name' in fields: - res['name'] = _('Purge data') - return res - - def find(self, cr, uid, context=None): - """ - Collect all rows from ir_model_data that refer - to a nonexisting model, or to a nonexisting - row in the model's table. - """ - res = [] - data_pool = self.pool['ir.model.data'] - data_ids = [] - unknown_models = [] - cr.execute("""SELECT DISTINCT(model) FROM ir_model_data""") - for (model,) in cr.fetchall(): - if not model: - continue - if not self.pool.get(model): - unknown_models.append(model) - continue - cr.execute( - """ - SELECT id FROM ir_model_data - WHERE model = %%s - AND res_id IS NOT NULL - AND NOT EXISTS ( - SELECT id FROM %s WHERE id=ir_model_data.res_id) - """ % self.pool[model]._table, (model,)) - data_ids += [data_row[0] for data_row in cr.fetchall()] - data_ids += data_pool.search( - cr, uid, [('model', 'in', unknown_models)], context=context) - for data in data_pool.browse(cr, uid, data_ids, context=context): - res.append((0, 0, { - 'data_id': data.id, - 'name': "%s.%s, object of type %s" % ( - data.module, data.name, data.model)})) - if not res: - raise orm.except_orm( - _('Nothing to do'), - _('No orphaned data entries found')) - return res - - _columns = { - 'purge_line_ids': fields.one2many( - 'cleanup.purge.line.data', - 'wizard_id', 'Data to purge'), - } diff --git a/database_cleanup/model/purge_menus.py b/database_cleanup/model/purge_menus.py deleted file mode 100644 index a9a336fe9..000000000 --- a/database_cleanup/model/purge_menus.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# OpenERP, Open Source Management Solution -# This module copyright (C) 2015 Therp BV (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## -from openerp.osv import orm, fields -from openerp.tools.translate import _ - - -class CleanupPurgeLineMenu(orm.TransientModel): - _inherit = 'cleanup.purge.line' - _name = 'cleanup.purge.line.menu' - - _columns = { - 'wizard_id': fields.many2one( - 'cleanup.purge.wizard.menu', 'Purge Wizard', readonly=True), - 'menu_id': fields.many2one('ir.ui.menu', 'Menu entry'), - } - - def purge(self, cr, uid, ids, context=None): - self.pool['ir.ui.menu'].unlink( - cr, uid, - [this.menu_id.id for this in self.browse(cr, uid, ids, - context=context)], - context=context) - return self.write(cr, uid, ids, {'purged': True}, context=context) - - -class CleanupPurgeWizardMenu(orm.TransientModel): - _inherit = 'cleanup.purge.wizard' - _name = 'cleanup.purge.wizard.menu' - - def default_get(self, cr, uid, fields, context=None): - res = super(CleanupPurgeWizardMenu, self).default_get( - cr, uid, fields, context=context) - if 'name' in fields: - res['name'] = _('Purge menus') - return res - - def find(self, cr, uid, context=None): - """ - Search for models that cannot be instantiated. - """ - res = [] - for menu in self.pool['ir.ui.menu'].browse( - cr, uid, self.pool['ir.ui.menu'].search( - cr, uid, [], context=dict( - context or {}, active_test=False))): - if not menu.action or menu.action.type != 'ir.actions.act_window': - continue - if not self.pool.get(menu.action.res_model): - res.append((0, 0, { - 'name': menu.complete_name, - 'menu_id': menu.id, - })) - if not res: - raise orm.except_orm( - _('Nothing to do'), - _('No dangling menu entries found')) - return res - - _columns = { - 'purge_line_ids': fields.one2many( - 'cleanup.purge.line.menu', - 'wizard_id', 'Menus to purge'), - } diff --git a/database_cleanup/model/purge_models.py b/database_cleanup/model/purge_models.py deleted file mode 100644 index 415f10489..000000000 --- a/database_cleanup/model/purge_models.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# OpenERP, Open Source Management Solution -# This module copyright (C) 2014 Therp BV (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - -from openerp.osv import orm, fields -from openerp.tools.translate import _ -from openerp.addons.base.ir.ir_model import MODULE_UNINSTALL_FLAG - - -class IrModel(orm.Model): - _inherit = 'ir.model' - - def _drop_table(self, cr, uid, ids, context=None): - # Allow to skip this step during model unlink - # The super method crashes if the model cannot be instantiated - if context and context.get('no_drop_table'): - return True - return super(IrModel, self)._drop_table(cr, uid, ids, context=context) - - def _inherited_models(self, cr, uid, ids, field_name, arg, context=None): - """this function crashes for undefined models""" - result = dict((i, []) for i in ids) - existing_model_ids = [ - this.id for this in self.browse(cr, uid, ids, context=context) - if self.pool.get(this.model) - ] - super_result = super(IrModel, self)._inherited_models( - cr, uid, existing_model_ids, field_name, arg, context=context) - result.update(super_result) - return result - - def _register_hook(self, cr): - # patch the function field instead of overwriting it - if self._columns['inherited_model_ids']._fnct !=\ - self._inherited_models.__func__: - self._columns['inherited_model_ids']._fnct =\ - self._inherited_models.__func__ - return super(IrModel, self)._register_hook(cr) - - -class CleanupPurgeLineModel(orm.TransientModel): - _inherit = 'cleanup.purge.line' - _name = 'cleanup.purge.line.model' - - _columns = { - 'wizard_id': fields.many2one( - 'cleanup.purge.wizard.model', 'Purge Wizard', readonly=True), - } - - def purge(self, cr, uid, ids, context=None): - """ - Unlink models upon manual confirmation. - """ - model_pool = self.pool['ir.model'] - attachment_pool = self.pool['ir.attachment'] - constraint_pool = self.pool['ir.model.constraint'] - fields_pool = self.pool['ir.model.fields'] - relation_pool = self.pool['ir.model.relation'] - - local_context = (context or {}).copy() - local_context.update({ - MODULE_UNINSTALL_FLAG: True, - 'no_drop_table': True, - }) - - for line in self.browse(cr, uid, ids, context=context): - cr.execute( - "SELECT id, model from ir_model WHERE model = %s", - (line.name,)) - row = cr.fetchone() - if row: - self.logger.info('Purging model %s', row[1]) - attachment_ids = attachment_pool.search( - cr, uid, [('res_model', '=', line.name)], context=context) - if attachment_ids: - cr.execute( - "UPDATE ir_attachment SET res_model = FALSE " - "WHERE id in %s", - (tuple(attachment_ids), )) - constraint_ids = constraint_pool.search( - cr, uid, [('model', '=', line.name)], context=context) - if constraint_ids: - constraint_pool.unlink( - cr, uid, constraint_ids, context=context) - relation_ids = fields_pool.search( - cr, uid, [('relation', '=', row[1])], context=context) - for relation in relation_ids: - try: - # Fails if the model on the target side - # cannot be instantiated - fields_pool.unlink(cr, uid, [relation], - context=local_context) - except KeyError: - pass - except AttributeError: - pass - relation_ids = relation_pool.search( - cr, uid, [('model', '=', line.name)], context=context) - for relation in relation_ids: - relation_pool.unlink(cr, uid, [relation], - context=local_context) - model_pool.unlink(cr, uid, [row[0]], context=local_context) - line.write({'purged': True}) - cr.commit() - return True - - -class CleanupPurgeWizardModel(orm.TransientModel): - _inherit = 'cleanup.purge.wizard' - _name = 'cleanup.purge.wizard.model' - - def default_get(self, cr, uid, fields, context=None): - res = super(CleanupPurgeWizardModel, self).default_get( - cr, uid, fields, context=context) - if 'name' in fields: - res['name'] = _('Purge models') - return res - - def find(self, cr, uid, context=None): - """ - Search for models that cannot be instantiated. - """ - res = [] - cr.execute("SELECT model from ir_model") - for (model,) in cr.fetchall(): - if not self.pool.get(model): - res.append((0, 0, {'name': model})) - if not res: - raise orm.except_orm( - _('Nothing to do'), - _('No orphaned models found')) - return res - - _columns = { - 'purge_line_ids': fields.one2many( - 'cleanup.purge.line.model', - 'wizard_id', 'Models to purge'), - } diff --git a/database_cleanup/model/purge_modules.py b/database_cleanup/model/purge_modules.py deleted file mode 100644 index d371b5f05..000000000 --- a/database_cleanup/model/purge_modules.py +++ /dev/null @@ -1,129 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# OpenERP, Open Source Management Solution -# This module copyright (C) 2014 Therp BV (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - -from openerp import pooler -from openerp.osv import orm, fields -from openerp.modules.module import get_module_path -from openerp.tools.translate import _ -from openerp.addons.base.ir.ir_model import MODULE_UNINSTALL_FLAG - - -class IrModelConstraint(orm.Model): - _inherit = 'ir.model.constraint' - - def _module_data_uninstall(self, cr, uid, ids, context=None): - """this function crashes for constraints on undefined models""" - for this in self.browse(cr, uid, ids, context=context): - if not self.pool.get(this.model.model): - ids.remove(this.id) - this.unlink() - return super(IrModelConstraint, self)._module_data_uninstall( - cr, uid, ids, context=context) - - -class IrModelData(orm.Model): - _inherit = 'ir.model.data' - - def _module_data_uninstall(self, cr, uid, modules_to_remove, context=None): - """this function crashes for xmlids on undefined models or fields - referring to undefined models""" - if context is None: - context = {} - ids = self.search(cr, uid, [('module', 'in', modules_to_remove)]) - for this in self.browse(cr, uid, ids, context=context): - if this.model == 'ir.model.fields': - ctx = context.copy() - ctx[MODULE_UNINSTALL_FLAG] = True - field = self.pool[this.model].browse( - cr, uid, this.res_id, context=ctx) - if not self.pool.get(field.model): - this.unlink() - continue - if not self.pool.get(this.model): - this.unlink() - return super(IrModelData, self)._module_data_uninstall( - cr, uid, modules_to_remove, context=context) - - -class CleanupPurgeLineModule(orm.TransientModel): - _inherit = 'cleanup.purge.line' - _name = 'cleanup.purge.line.module' - - _columns = { - 'wizard_id': fields.many2one( - 'cleanup.purge.wizard.module', 'Purge Wizard', readonly=True), - } - - def purge(self, cr, uid, ids, context=None): - """ - Uninstall modules upon manual confirmation, then reload - the database. - """ - module_pool = self.pool['ir.module.module'] - lines = self.browse(cr, uid, ids, context=context) - module_names = [line.name for line in lines if not line.purged] - module_ids = module_pool.search( - cr, uid, [('name', 'in', module_names)], context=context) - if not module_ids: - return True - self.logger.info('Purging modules %s', ', '.join(module_names)) - module_pool.write( - cr, uid, module_ids, {'state': 'to remove'}, context=context) - cr.commit() - _db, _pool = pooler.restart_pool(cr.dbname, update_module=True) - module_pool.unlink(cr, uid, module_ids, context=context) - return self.write(cr, uid, ids, {'purged': True}, context=context) - - -class CleanupPurgeWizardModule(orm.TransientModel): - _inherit = 'cleanup.purge.wizard' - _name = 'cleanup.purge.wizard.module' - - def default_get(self, cr, uid, fields, context=None): - res = super(CleanupPurgeWizardModule, self).default_get( - cr, uid, fields, context=context) - if 'name' in fields: - res['name'] = _('Purge modules') - return res - - def find(self, cr, uid, context=None): - module_pool = self.pool['ir.module.module'] - module_ids = module_pool.search(cr, uid, [], context=context) - res = [] - for module in module_pool.browse(cr, uid, module_ids, context=context): - if get_module_path(module.name): - continue - if module.state == 'uninstalled': - module_pool.unlink(cr, uid, module.id, context=context) - continue - res.append((0, 0, {'name': module.name})) - - if not res: - raise orm.except_orm( - _('Nothing to do'), - _('No modules found to purge')) - return res - - _columns = { - 'purge_line_ids': fields.one2many( - 'cleanup.purge.line.module', - 'wizard_id', 'Modules to purge'), - } diff --git a/database_cleanup/model/purge_tables.py b/database_cleanup/model/purge_tables.py deleted file mode 100644 index 50bf3d8e6..000000000 --- a/database_cleanup/model/purge_tables.py +++ /dev/null @@ -1,138 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# OpenERP, Open Source Management Solution -# This module copyright (C) 2014 Therp BV (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - -from openerp.osv import orm, fields -from openerp.tools.translate import _ - - -class CleanupPurgeLineTable(orm.TransientModel): - _inherit = 'cleanup.purge.line' - _name = 'cleanup.purge.line.table' - - _columns = { - 'wizard_id': fields.many2one( - 'cleanup.purge.wizard.table', 'Purge Wizard', readonly=True), - } - - def purge(self, cr, uid, ids, context=None): - """ - Unlink tables upon manual confirmation. - """ - lines = self.browse(cr, uid, ids, context=context) - tables = [line.name for line in lines] - for line in lines: - if line.purged: - continue - - # Retrieve constraints on the tables to be dropped - # This query is referenced in numerous places - # on the Internet but credits probably go to Tom Lane - # in this post http://www.postgresql.org/\ - # message-id/22895.1226088573@sss.pgh.pa.us - # Only using the constraint name and the source table, - # but I'm leaving the rest in for easier debugging - cr.execute( - """ - SELECT conname, confrelid::regclass, af.attname AS fcol, - conrelid::regclass, a.attname AS col - FROM pg_attribute af, pg_attribute a, - (SELECT conname, conrelid, confrelid,conkey[i] AS conkey, - confkey[i] AS confkey - FROM (select conname, conrelid, confrelid, conkey, - confkey, generate_series(1,array_upper(conkey,1)) AS i - FROM pg_constraint WHERE contype = 'f') ss) ss2 - WHERE af.attnum = confkey AND af.attrelid = confrelid AND - a.attnum = conkey AND a.attrelid = conrelid - AND confrelid::regclass = '%s'::regclass; - """ % line.name) - - for constraint in cr.fetchall(): - if constraint[3] in tables: - self.logger.info( - 'Dropping constraint %s on table %s (to be dropped)', - constraint[0], constraint[3]) - cr.execute( - "ALTER TABLE %s DROP CONSTRAINT %s" % ( - constraint[3], constraint[0])) - - self.logger.info( - 'Dropping table %s', line.name) - cr.execute("DROP TABLE \"%s\"" % (line.name,)) - line.write({'purged': True}) - cr.commit() - return True - - -class CleanupPurgeWizardTable(orm.TransientModel): - _inherit = 'cleanup.purge.wizard' - _name = 'cleanup.purge.wizard.table' - - def default_get(self, cr, uid, fields, context=None): - res = super(CleanupPurgeWizardTable, self).default_get( - cr, uid, fields, context=context) - if 'name' in fields: - res['name'] = _('Purge tables') - return res - - def find(self, cr, uid, context=None): - """ - Search for tables that cannot be instantiated. - Ignore views for now. - """ - model_ids = self.pool['ir.model'].search(cr, uid, [], context=context) - # Start out with known tables with no model - known_tables = ['wkf_witm_trans'] - for model in self.pool['ir.model'].browse( - cr, uid, model_ids, context=context): - - model_pool = self.pool.get(model.model) - if not model_pool: - continue - known_tables.append(model_pool._table) - known_tables += [ - column._sql_names(model_pool)[0] - for column in model_pool._columns.values() - if (column._type == 'many2many' and - hasattr(column, '_rel')) # unstored function fields of - # type m2m don't have _rel - ] - - # Cannot pass table names as a psycopg argument - known_tables_repr = ",".join( - [("'%s'" % table) for table in known_tables]) - cr.execute( - """ - SELECT table_name FROM information_schema.tables - WHERE table_schema = 'public' AND table_type = 'BASE TABLE' - AND table_name NOT IN (%s)""" % known_tables_repr) - - res = [(0, 0, {'name': row[0]}) for row in cr.fetchall()] - if not res: - raise orm.except_orm( - _('Nothing to do'), - _('No orphaned tables found')) - return res - - _columns = { - 'purge_line_ids': fields.one2many( - 'cleanup.purge.line.table', - 'wizard_id', 'Tables to purge'), - } diff --git a/database_cleanup/model/purge_wizard.py b/database_cleanup/model/purge_wizard.py deleted file mode 100644 index f02f5dbc2..000000000 --- a/database_cleanup/model/purge_wizard.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# OpenERP, Open Source Management Solution -# This module copyright (C) 2014 Therp BV (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - -import logging -from openerp.osv import orm, fields - - -class CleanupPurgeLine(orm.AbstractModel): - """ Abstract base class for the purge wizard lines """ - _name = 'cleanup.purge.line' - _order = 'name' - _columns = { - 'name': fields.char('Name', size=256, readonly=True), - 'purged': fields.boolean('Purged', readonly=True), - } - - logger = logging.getLogger('openerp.addons.database_cleanup') - - def purge(self, cr, uid, ids, context=None): - raise NotImplementedError - - -class PurgeWizard(orm.AbstractModel): - """ Abstract base class for the purge wizards """ - _name = 'cleanup.purge.wizard' - - def default_get(self, cr, uid, fields, context=None): - res = super(PurgeWizard, self).default_get( - cr, uid, fields, context=context) - if 'purge_line_ids' in fields: - res['purge_line_ids'] = self.find(cr, uid, context=None) - return res - - def find(self, cr, uid, ids, context=None): - raise NotImplementedError - - def purge_all(self, cr, uid, ids, context=None): - line_pool = self.pool[self._columns['purge_line_ids']._obj] - for wizard in self.browse(cr, uid, ids, context=context): - line_pool.purge( - cr, uid, [line.id for line in wizard.purge_line_ids], - context=context) - return True - - def get_wizard_action(self, cr, uid, context=None): - wizard_id = self.create(cr, uid, {}, context=context) - return { - 'type': 'ir.actions.act_window', - 'views': [(False, 'form')], - 'res_model': self._name, - 'res_id': wizard_id, - 'flags': { - 'action_buttons': False, - 'sidebar': False, - }, - } - - def select_lines(self, cr, uid, ids, context=None): - return { - 'type': 'ir.actions.act_window', - 'name': 'Select lines to purge', - 'views': [(False, 'tree'), (False, 'form')], - 'res_model': self._columns['purge_line_ids']._obj, - 'domain': [('wizard_id', 'in', ids)], - } - - _columns = { - 'name': fields.char('Name', size=64, readonly=True), - } diff --git a/database_cleanup/model/__init__.py b/database_cleanup/models/__init__.py similarity index 100% rename from database_cleanup/model/__init__.py rename to database_cleanup/models/__init__.py diff --git a/database_cleanup/models/purge_columns.py b/database_cleanup/models/purge_columns.py new file mode 100644 index 000000000..3a3718c21 --- /dev/null +++ b/database_cleanup/models/purge_columns.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# © 2014-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from openerp import _, api, fields, models +from openerp.exceptions import UserError +from ..identifier_adapter import IdentifierAdapter + + +class CleanupPurgeLineColumn(models.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.column' + + model_id = fields.Many2one('ir.model', 'Model', required=True, + ondelete='CASCADE') + wizard_id = fields.Many2one( + 'cleanup.purge.wizard.column', 'Purge Wizard', readonly=True) + + @api.multi + def purge(self): + """ + Unlink columns upon manual confirmation. + """ + for line in self: + if line.purged: + continue + model_pool = self.env[line.model_id.model] + # Check whether the column actually still exists. + # Inheritance such as stock.picking.in from stock.picking + # can lead to double attempts at removal + self.env.cr.execute( + 'SELECT count(attname) FROM pg_attribute ' + 'WHERE attrelid = ' + '( SELECT oid FROM pg_class WHERE relname = %s ) ' + 'AND attname = %s', + (model_pool._table, line.name)) + if not self.env.cr.fetchone()[0]: + continue + + self.logger.info( + 'Dropping column %s from table %s', + line.name, model_pool._table) + self.env.cr.execute( + 'ALTER TABLE %s DROP COLUMN %s', + ( + IdentifierAdapter(model_pool._table), + IdentifierAdapter(line.name) + )) + line.write({'purged': True}) + # we need this commit because the ORM will deadlock if + # we still have a pending transaction + self.env.cr.commit() # pylint: disable=invalid-commit + return True + + +class CleanupPurgeWizardColumn(models.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.column' + _description = 'Purge columns' + + # List of known columns in use without corresponding fields + # Format: {table: [fields]} + blacklist = { + 'wkf_instance': ['uid'], # lp:1277899 + } + + @api.model + def get_orphaned_columns(self, model_pools): + """ + From openobject-server/openerp/osv/orm.py + Iterate on the database columns to identify columns + of fields which have been removed + """ + columns = list(set([ + column + for model_pool in model_pools + for column in model_pool._columns + if not (isinstance(model_pool._columns[column], + fields.fields.function) and + not model_pool._columns[column].store) + ])) + columns += models.MAGIC_COLUMNS + columns += self.blacklist.get(model_pools[0]._table, []) + + self.env.cr.execute( + "SELECT a.attname FROM pg_class c, pg_attribute a " + "WHERE c.relname=%s AND c.oid=a.attrelid AND a.attisdropped=False " + "AND pg_catalog.format_type(a.atttypid, a.atttypmod) " + "NOT IN ('cid', 'tid', 'oid', 'xid') " + "AND a.attname NOT IN %s", + (model_pools[0]._table, tuple(columns))) + return [column for column, in self.env.cr.fetchall()] + + @api.model + def find(self): + """ + Search for columns that are not in the corresponding model. + + Group models by table to prevent false positives for columns + that are only in some of the models sharing the same table. + Example of this is 'sale_id' not being a field of stock.picking.in + """ + res = [] + + # mapping of tables to tuples (model id, [pool1, pool2, ...]) + table2model = {} + + for model in self.env['ir.model'].search([]): + if model.model not in self.env: + continue + model_pool = self.env[model.model] + if not model_pool._auto: + continue + table2model.setdefault( + model_pool._table, (model.id, []) + )[1].append(model_pool) + + for table, model_spec in table2model.iteritems(): + for column in self.get_orphaned_columns(model_spec[1]): + res.append((0, 0, { + 'name': column, + 'model_id': model_spec[0]})) + if not res: + raise UserError(_('No orphaned columns found')) + return res + + purge_line_ids = fields.One2many( + 'cleanup.purge.line.column', 'wizard_id', 'Columns to purge') diff --git a/database_cleanup/models/purge_data.py b/database_cleanup/models/purge_data.py new file mode 100644 index 000000000..9e1817ff8 --- /dev/null +++ b/database_cleanup/models/purge_data.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# © 2014-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from openerp import _, api, fields, models +from openerp.exceptions import UserError +from ..identifier_adapter import IdentifierAdapter + + +class CleanupPurgeLineData(models.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.data' + + data_id = fields.Many2one('ir.model.data', 'Data entry') + wizard_id = fields.Many2one( + 'cleanup.purge.wizard.data', 'Purge Wizard', readonly=True) + + @api.multi + def purge(self): + """Unlink data entries upon manual confirmation.""" + to_unlink = self.filtered(lambda x: not x.purged and x.data_id) + self.logger.info('Purging data entries: %s', to_unlink.mapped('name')) + to_unlink.mapped('data_id').unlink() + return self.write({'purged': True}) + + +class CleanupPurgeWizardData(models.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.data' + _description = 'Purge data' + + @api.model + def find(self): + """Collect all rows from ir_model_data that refer + to a nonexisting model, or to a nonexisting + row in the model's table.""" + res = [] + data_ids = [] + unknown_models = [] + self.env.cr.execute("""SELECT DISTINCT(model) FROM ir_model_data""") + for model, in self.env.cr.fetchall(): + if not model: + continue + if model not in self.env: + unknown_models.append(model) + continue + self.env.cr.execute( + """ + SELECT id FROM ir_model_data + WHERE model = %s + AND res_id IS NOT NULL + AND NOT EXISTS ( + SELECT id FROM %s WHERE id=ir_model_data.res_id) + """, (model, IdentifierAdapter(self.env[model]._table))) + data_ids.extend(data_row for data_row, in self.env.cr.fetchall()) + data_ids += self.env['ir.model.data'].search([ + ('model', 'in', unknown_models), + ]).ids + for data in self.env['ir.model.data'].browse(data_ids): + res.append((0, 0, { + 'data_id': data.id, + 'name': "%s.%s, object of type %s" % ( + data.module, data.name, data.model)})) + if not res: + raise UserError(_('No orphaned data entries found')) + return res + + purge_line_ids = fields.One2many( + 'cleanup.purge.line.data', 'wizard_id', 'Data to purge') diff --git a/database_cleanup/models/purge_menus.py b/database_cleanup/models/purge_menus.py new file mode 100644 index 000000000..bef098b28 --- /dev/null +++ b/database_cleanup/models/purge_menus.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# © 2014-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from openerp import _, api, fields, models +from openerp.exceptions import UserError + + +class CleanupPurgeLineMenu(models.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.menu' + + wizard_id = fields.Many2one( + 'cleanup.purge.wizard.menu', 'Purge Wizard', readonly=True) + menu_id = fields.Many2one('ir.ui.menu', 'Menu entry') + + @api.multi + def purge(self): + self.mapped('menu_id').unlink() + return self.write({'purged': True}) + + +class CleanupPurgeWizardMenu(models.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.menu' + _description = 'Purge menus' + + @api.model + def find(self): + """ + Search for models that cannot be instantiated. + """ + res = [] + for menu in self.env['ir.ui.menu'].with_context(active_test=False)\ + .search([('action', '!=', False)]): + if menu.action.type != 'ir.actions.act_window': + continue + if menu.action.res_model not in self.env or\ + menu.action.src_model not in self.env: + res.append((0, 0, { + 'name': menu.complete_name, + 'menu_id': menu.id, + })) + if not res: + raise UserError(_('No dangling menu entries found')) + return res + + purge_line_ids = fields.One2many( + 'cleanup.purge.line.menu', 'wizard_id', 'Menus to purge') diff --git a/database_cleanup/models/purge_models.py b/database_cleanup/models/purge_models.py new file mode 100644 index 000000000..996cea61d --- /dev/null +++ b/database_cleanup/models/purge_models.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# © 2014-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from openerp import _, api, models, fields +from openerp.exceptions import UserError +from openerp.addons.base.ir.ir_model import MODULE_UNINSTALL_FLAG + + +class IrModel(models.Model): + _inherit = 'ir.model' + + @api.multi + def _drop_table(self): + # Allow to skip this step during model unlink + # The super method crashes if the model cannot be instantiated + if self.env.context.get('no_drop_table'): + return True + return super(IrModel, self)._drop_table() + + @api.multi + def _inherited_models(self, field_name, arg): + """this function crashes for undefined models""" + result = dict((i, []) for i in self.ids) + existing_model_ids = [ + this.id for this in self if this.model in self.env + ] + super_result = super(IrModel, self.browse(existing_model_ids))\ + ._inherited_models(field_name, arg) + result.update(super_result) + return result + + def _register_hook(self, cr): + # patch the function field instead of overwriting it + if self._columns['inherited_model_ids']._fnct !=\ + self._inherited_models.__func__: + self._columns['inherited_model_ids']._fnct =\ + self._inherited_models.__func__ + return super(IrModel, self)._register_hook(cr) + + +class CleanupPurgeLineModel(models.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.model' + _description = 'Purge models' + + wizard_id = fields.Many2one( + 'cleanup.purge.wizard.model', 'Purge Wizard', readonly=True) + + @api.multi + def purge(self): + """ + Unlink models upon manual confirmation. + """ + context_flags = { + MODULE_UNINSTALL_FLAG: True, + 'no_drop_table': True, + } + + for line in self: + self.env.cr.execute( + "SELECT id, model from ir_model WHERE model = %s", + (line.name,)) + row = self.env.cr.fetchone() + if not row: + continue + self.logger.info('Purging model %s', row[1]) + attachments = self.env['ir.attachment'].search([ + ('res_model', '=', line.name) + ]) + if attachments: + self.env.cr.execute( + "UPDATE ir_attachment SET res_model = NULL " + "WHERE id in %s", + (tuple(attachments.ids), )) + self.env['ir.model.constraint'].search([ + ('model', '=', line.name), + ]).unlink() + relations = self.env['ir.model.fields'].search([ + ('relation', '=', row[1]), + ]).with_context(**context_flags) + for relation in relations: + try: + # Fails if the model on the target side + # cannot be instantiated + relation.unlink() + except KeyError: + pass + except AttributeError: + pass + self.env['ir.model.relation'].search([ + ('model', '=', line.name) + ]).with_context(**context_flags).unlink() + self.env['ir.model'].browse([row[0]])\ + .with_context(**context_flags).unlink() + line.write({'purged': True}) + return True + + +class CleanupPurgeWizardModel(models.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.model' + _description = 'Purge models' + + @api.model + def find(self): + """ + Search for models that cannot be instantiated. + """ + res = [] + self.env.cr.execute("SELECT model from ir_model") + for model, in self.env.cr.fetchall(): + if model not in self.env: + res.append((0, 0, {'name': model})) + if not res: + raise UserError(_('No orphaned models found')) + return res + + purge_line_ids = fields.One2many( + 'cleanup.purge.line.model', 'wizard_id', 'Models to purge') diff --git a/database_cleanup/models/purge_modules.py b/database_cleanup/models/purge_modules.py new file mode 100644 index 000000000..9664d1a21 --- /dev/null +++ b/database_cleanup/models/purge_modules.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# © 2014-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from openerp import _, api, fields, models +from openerp.exceptions import UserError +from openerp.modules.registry import RegistryManager +from openerp.modules.module import get_module_path +from openerp.addons.base.ir.ir_model import MODULE_UNINSTALL_FLAG + + +class IrModelData(models.Model): + _inherit = 'ir.model.data' + + @api.model + def _module_data_uninstall(self, modules_to_remove): + """this function crashes for xmlids on undefined models or fields + referring to undefined models""" + for this in self.search([('module', 'in', modules_to_remove)]): + if this.model == 'ir.model.fields': + field = self.env[this.model].with_context( + **{MODULE_UNINSTALL_FLAG: True}).browse(this.res_id) + if field.model not in self.env: + this.unlink() + continue + if this.model not in self.env: + this.unlink() + return super(IrModelData, self)._module_data_uninstall( + modules_to_remove) + + +class CleanupPurgeLineModule(models.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.module' + + wizard_id = fields.Many2one( + 'cleanup.purge.wizard.module', 'Purge Wizard', readonly=True) + + @api.multi + def purge(self): + """ + Uninstall modules upon manual confirmation, then reload + the database. + """ + module_names = self.filtered(lambda x: not x.purged).mapped('name') + modules = self.env['ir.module.module'].search([ + ('name', 'in', module_names) + ]) + if not modules: + return True + self.logger.info('Purging modules %s', ', '.join(module_names)) + modules.write({'state': 'to remove'}) + # we need this commit because reloading the registry would roll back + # our changes + self.env.cr.commit() # pylint: disable=invalid-commit + RegistryManager.new(self.env.cr.dbname, update_module=True) + modules.unlink() + return self.write({'purged': True}) + + +class CleanupPurgeWizardModule(models.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.module' + _description = 'Purge modules' + + @api.model + def find(self): + res = [] + for module in self.env['ir.module.module'].search([]): + if get_module_path(module.name): + continue + if module.state == 'uninstalled': + module.unlink() + continue + res.append((0, 0, {'name': module.name})) + + if not res: + raise UserError(_('No modules found to purge')) + return res + + purge_line_ids = fields.One2many( + 'cleanup.purge.line.module', 'wizard_id', 'Modules to purge') diff --git a/database_cleanup/models/purge_tables.py b/database_cleanup/models/purge_tables.py new file mode 100644 index 000000000..62503f790 --- /dev/null +++ b/database_cleanup/models/purge_tables.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# © 2014-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from openerp import api, fields, models, _ +from openerp.exceptions import UserError +from ..identifier_adapter import IdentifierAdapter + + +class CleanupPurgeLineTable(models.TransientModel): + _inherit = 'cleanup.purge.line' + _name = 'cleanup.purge.line.table' + + wizard_id = fields.Many2one( + 'cleanup.purge.wizard.table', 'Purge Wizard', readonly=True) + + @api.multi + def purge(self): + """ + Unlink tables upon manual confirmation. + """ + tables = self.mapped('name') + for line in self: + if line.purged: + continue + + # Retrieve constraints on the tables to be dropped + # This query is referenced in numerous places + # on the Internet but credits probably go to Tom Lane + # in this post http://www.postgresql.org/\ + # message-id/22895.1226088573@sss.pgh.pa.us + # Only using the constraint name and the source table, + # but I'm leaving the rest in for easier debugging + self.env.cr.execute( + """ + SELECT conname, confrelid::regclass, af.attname AS fcol, + conrelid::regclass, a.attname AS col + FROM pg_attribute af, pg_attribute a, + (SELECT conname, conrelid, confrelid,conkey[i] AS conkey, + confkey[i] AS confkey + FROM (select conname, conrelid, confrelid, conkey, + confkey, generate_series(1,array_upper(conkey,1)) AS i + FROM pg_constraint WHERE contype = 'f') ss) ss2 + WHERE af.attnum = confkey AND af.attrelid = confrelid AND + a.attnum = conkey AND a.attrelid = conrelid + AND confrelid::regclass = '%s'::regclass; + """, (IdentifierAdapter(line.name, quote=False),)) + + for constraint in self.env.cr.fetchall(): + if constraint[3] in tables: + self.logger.info( + 'Dropping constraint %s on table %s (to be dropped)', + constraint[0], constraint[3]) + self.env.cr.execute( + "ALTER TABLE %s DROP CONSTRAINT %s", + ( + IdentifierAdapter(constraint[3]), + IdentifierAdapter(constraint[0]) + )) + + self.logger.info( + 'Dropping table %s', line.name) + self.env.cr.execute( + "DROP TABLE %s", (IdentifierAdapter(line.name),)) + line.write({'purged': True}) + return True + + +class CleanupPurgeWizardTable(models.TransientModel): + _inherit = 'cleanup.purge.wizard' + _name = 'cleanup.purge.wizard.table' + _description = 'Purge tables' + + @api.model + def find(self): + """ + Search for tables that cannot be instantiated. + Ignore views for now. + """ + # Start out with known tables with no model + known_tables = ['wkf_witm_trans'] + for model in self.env['ir.model'].search([]): + if model.model not in self.env: + continue + model_pool = self.env[model.model] + known_tables.append(model_pool._table) + known_tables += [ + column._sql_names(model_pool)[0] + for column in model_pool._columns.values() + if (column._type == 'many2many' and + hasattr(column, '_rel')) # unstored function fields of + # type m2m don't have _rel + ] + + self.env.cr.execute( + """ + SELECT table_name FROM information_schema.tables + WHERE table_schema = 'public' AND table_type = 'BASE TABLE' + AND table_name NOT IN %s""", (tuple(known_tables),)) + + res = [(0, 0, {'name': row[0]}) for row in self.env.cr.fetchall()] + if not res: + raise UserError(_('No orphaned tables found')) + return res + + purge_line_ids = fields.One2many( + 'cleanup.purge.line.table', 'wizard_id', 'Tables to purge') diff --git a/database_cleanup/models/purge_wizard.py b/database_cleanup/models/purge_wizard.py new file mode 100644 index 000000000..aa44ffe84 --- /dev/null +++ b/database_cleanup/models/purge_wizard.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# © 2014-2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import logging +from openerp import _, api, fields, models +from openerp.exceptions import AccessDenied + + +class CleanupPurgeLine(models.AbstractModel): + """ Abstract base class for the purge wizard lines """ + _name = 'cleanup.purge.line' + _order = 'name' + + name = fields.Char('Name', readonly=True) + purged = fields.Boolean('Purged', readonly=True) + wizard_id = fields.Many2one('cleanup.purge.wizard') + + logger = logging.getLogger('openerp.addons.database_cleanup') + + @api.multi + def purge(self): + raise NotImplementedError + + @api.model + def create(self, values): + # make sure the user trying this is actually supposed to do it + if not self.env.ref('database_cleanup.menu_database_cleanup')\ + .parent_id._filter_visible_menus(): + raise AccessDenied + return super(CleanupPurgeLine, self).create(values) + + +class PurgeWizard(models.AbstractModel): + """ Abstract base class for the purge wizards """ + _name = 'cleanup.purge.wizard' + _description = 'Purge stuff' + + @api.model + def default_get(self, fields_list): + res = super(PurgeWizard, self).default_get(fields_list) + if 'purge_line_ids' in fields_list: + res['purge_line_ids'] = self.find() + return res + + @api.multi + def find(self): + raise NotImplementedError + + @api.multi + def purge_all(self): + self.mapped('purge_line_ids').purge() + return True + + @api.model + def get_wizard_action(self): + wizard = self.create({}) + return { + 'type': 'ir.actions.act_window', + 'name': wizard.display_name, + 'views': [(False, 'form')], + 'res_model': self._name, + 'res_id': wizard.id, + 'flags': { + 'action_buttons': False, + 'sidebar': False, + }, + } + + @api.multi + def select_lines(self): + return { + 'type': 'ir.actions.act_window', + 'name': _('Select lines to purge'), + 'views': [(False, 'tree'), (False, 'form')], + 'res_model': self._fields['purge_line_ids'].comodel_name, + 'domain': [('wizard_id', 'in', self.ids)], + } + + @api.multi + def name_get(self): + return [ + (this.id, self._description) + for this in self + ] + + @api.model + def create(self, values): + # make sure the user trying this is actually supposed to do it + if not self.env.ref('database_cleanup.menu_database_cleanup')\ + .parent_id._filter_visible_menus(): + raise AccessDenied + return super(PurgeWizard, self).create(values) + + purge_line_ids = fields.One2many('cleanup.purge.line', 'wizard_id') diff --git a/database_cleanup/tests/__init__.py b/database_cleanup/tests/__init__.py new file mode 100644 index 000000000..265497c37 --- /dev/null +++ b/database_cleanup/tests/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# © 2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from . import test_database_cleanup diff --git a/database_cleanup/tests/test_database_cleanup.py b/database_cleanup/tests/test_database_cleanup.py new file mode 100644 index 000000000..f04609f1c --- /dev/null +++ b/database_cleanup/tests/test_database_cleanup.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# © 2016 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from psycopg2 import ProgrammingError +from openerp.tools import config +from openerp.tests.common import TransactionCase + + +class TestDatabaseCleanup(TransactionCase): + def test_database_cleanup(self): + # create an orphaned column + self.cr.execute( + 'alter table res_users add column database_cleanup_test int') + purge_columns = self.env['cleanup.purge.wizard.column'].create({}) + purge_columns.purge_all() + # must be removed by the wizard + with self.assertRaises(ProgrammingError): + with self.registry.cursor() as cr: + cr.execute('select database_cleanup_test from res_users') + + # create a data entry pointing nowhere + self.cr.execute('select max(id) + 1 from res_users') + self.env['ir.model.data'].create({ + 'module': 'database_cleanup', + 'name': 'test_no_data_entry', + 'model': 'res.users', + 'res_id': self.cr.fetchone()[0], + }) + purge_data = self.env['cleanup.purge.wizard.data'].create({}) + purge_data.purge_all() + # must be removed by the wizard + with self.assertRaises(ValueError): + self.env.ref('database_cleanup.test_no_data_entry') + + # create a nonexistent model + self.env['ir.model'].create({ + 'name': 'Database cleanup test model', + 'model': 'x_database.cleanup.test.model', + }) + self.env.cr.execute( + 'insert into ir_attachment (name, res_model, res_id, type) values ' + "('test attachment', 'database.cleanup.test.model', 42, 'binary')") + self.registry.models.pop('x_database.cleanup.test.model') + self.registry._pure_function_fields.pop( + 'x_database.cleanup.test.model') + purge_models = self.env['cleanup.purge.wizard.model'].create({}) + purge_models.purge_all() + # must be removed by the wizard + self.assertFalse(self.env['ir.model'].search([ + ('model', '=', 'x_database.cleanup.test.model'), + ])) + + # create a nonexistent module + self.env['ir.module.module'].create({ + 'name': 'database_cleanup_test', + 'state': 'to upgrade', + }) + purge_modules = self.env['cleanup.purge.wizard.module'].create({}) + # this reloads our registry, and we don't want to run tests twice + config.options['test_enable'] = False + purge_modules.purge_all() + config.options['test_enable'] = True + # must be removed by the wizard + self.assertFalse(self.env['ir.module.module'].search([ + ('name', '=', 'database_cleanup_test'), + ])) + + # create an orphaned table + self.env.cr.execute('create table database_cleanup_test (test int)') + purge_tables = self.env['cleanup.purge.wizard.table'].create({}) + purge_tables.purge_all() + with self.assertRaises(ProgrammingError): + with self.registry.cursor() as cr: + self.env.cr.execute('select * from database_cleanup_test') diff --git a/database_cleanup/view/menu.xml b/database_cleanup/views/menu.xml similarity index 100% rename from database_cleanup/view/menu.xml rename to database_cleanup/views/menu.xml diff --git a/database_cleanup/view/purge_columns.xml b/database_cleanup/views/purge_columns.xml similarity index 54% rename from database_cleanup/view/purge_columns.xml rename to database_cleanup/views/purge_columns.xml index 0f335ea28..1264a8907 100644 --- a/database_cleanup/view/purge_columns.xml +++ b/database_cleanup/views/purge_columns.xml @@ -1,32 +1,14 @@ - - Form view for purge columns wizard cleanup.purge.wizard.column + + primary -
-

- -

-