Browse Source

[FIX] Don't remove uid field from wkf_instance, which is written in

raw SQL query (but never read afterwards). Workaround for
    lp:1277899

[FIX] Preserve dangling workflow table which is in use

[RFR] Group models per table when detecting columns to purge
      to prevent problems with models sharing the same table

[ADD] Allow purging of dangling data entries

[FIX] Data purging now working

[IMP] Docstrings

[FIX] Label
[FIX] Catch attempt to unlink field from nonexisting model

[RFR] Flake8
pull/1408/head
Stefan Rijnhart 11 years ago
committed by Martin Trigaux
parent
commit
f08a8e1024
  1. 5
      database_cleanup/__openerp__.py
  2. 1
      database_cleanup/model/__init__.py
  3. 43
      database_cleanup/model/purge_columns.py
  4. 106
      database_cleanup/model/purge_data.py
  5. 9
      database_cleanup/model/purge_models.py
  6. 11
      database_cleanup/model/purge_tables.py
  7. 1
      database_cleanup/model/purge_wizard.py
  8. 9
      database_cleanup/view/menu.xml
  9. 37
      database_cleanup/view/purge_data.xml

5
database_cleanup/__openerp__.py

@ -31,12 +31,13 @@
'view/purge_models.xml', 'view/purge_models.xml',
'view/purge_columns.xml', 'view/purge_columns.xml',
'view/purge_tables.xml', 'view/purge_tables.xml',
'view/purge_data.xml',
'view/menu.xml', 'view/menu.xml',
], ],
'description': """\ 'description': """\
Clean your OpenERP database from remnants of modules, models, columns and Clean your OpenERP database from remnants of modules, models, columns and
tables left by uninstalled modules (prior to 7.0) or a homebrew database upgrade
to a new major version of OpenERP.
tables left by uninstalled modules (prior to 7.0) or a homebrew database
upgrade to a new major version of OpenERP.
After installation of this module, go to the Settings menu -> Technical -> After installation of this module, go to the Settings menu -> Technical ->
Database cleanup. Go through the modules, models, columns and tables Database cleanup. Go through the modules, models, columns and tables

1
database_cleanup/model/__init__.py

@ -3,3 +3,4 @@ from . import purge_modules
from . import purge_models from . import purge_models
from . import purge_columns from . import purge_columns
from . import purge_tables from . import purge_tables
from . import purge_data

43
database_cleanup/model/purge_columns.py

@ -53,7 +53,7 @@ class CleanupPurgeLineColumn(orm.TransientModel):
'WHERE attrelid = ' 'WHERE attrelid = '
'( SELECT oid FROM pg_class WHERE relname = %s ) ' '( SELECT oid FROM pg_class WHERE relname = %s ) '
'AND attname = %s', 'AND attname = %s',
(model_pool._table, line.name));
(model_pool._table, line.name))
if not cr.fetchone()[0]: if not cr.fetchone()[0]:
continue continue
@ -68,10 +68,17 @@ class CleanupPurgeLineColumn(orm.TransientModel):
cr.commit() cr.commit()
return True return True
class CleanupPurgeWizardColumn(orm.TransientModel): class CleanupPurgeWizardColumn(orm.TransientModel):
_inherit = 'cleanup.purge.wizard' _inherit = 'cleanup.purge.wizard'
_name = 'cleanup.purge.wizard.column' _name = 'cleanup.purge.wizard.column'
# List of known columns in use without corresponding fields
# Format: {table: [fields]}
blacklist = {
'wkf_instance': ['uid'], # lp:1277899
}
def default_get(self, cr, uid, fields, context=None): def default_get(self, cr, uid, fields, context=None):
res = super(CleanupPurgeWizardColumn, self).default_get( res = super(CleanupPurgeWizardColumn, self).default_get(
cr, uid, fields, context=context) cr, uid, fields, context=context)
@ -79,17 +86,22 @@ class CleanupPurgeWizardColumn(orm.TransientModel):
res['name'] = _('Purge columns') res['name'] = _('Purge columns')
return res return res
def get_orphaned_columns(self, cr, uid, model_pool, context=None):
def get_orphaned_columns(self, cr, uid, model_pools, context=None):
""" """
From openobject-server/openerp/osv/orm.py From openobject-server/openerp/osv/orm.py
Iterate on the database columns to identify columns Iterate on the database columns to identify columns
of fields which have been removed of fields which have been removed
""" """
columns = [
c for c in model_pool._columns
if not (isinstance(model_pool._columns[c], fields.function)
and not model_pool._columns[c].store)]
columns = list(set([
column for model_pool in model_pools
for column in model_pool._columns
if not (isinstance(model_pool._columns[column], fields.function)
and not model_pool._columns[column].store)
]))
columns += orm.MAGIC_COLUMNS columns += orm.MAGIC_COLUMNS
columns += self.blacklist.get(model_pools[0]._table, [])
cr.execute("SELECT a.attname" cr.execute("SELECT a.attname"
" FROM pg_class c, pg_attribute a" " FROM pg_class c, pg_attribute a"
" WHERE c.relname=%s" " WHERE c.relname=%s"
@ -98,26 +110,37 @@ class CleanupPurgeWizardColumn(orm.TransientModel):
" AND pg_catalog.format_type(a.atttypid, a.atttypmod)" " AND pg_catalog.format_type(a.atttypid, a.atttypmod)"
" NOT IN ('cid', 'tid', 'oid', 'xid')" " NOT IN ('cid', 'tid', 'oid', 'xid')"
" AND a.attname NOT IN %s", " AND a.attname NOT IN %s",
(model_pool._table, False, tuple(columns))),
(model_pools[0]._table, False, tuple(columns))),
return [column[0] for column in cr.fetchall()] return [column[0] for column in cr.fetchall()]
def find(self, cr, uid, context=None): def find(self, cr, uid, context=None):
""" """
Search for columns that are not in the corresponding model. Search for columns that are not in the corresponding model.
Group models by table to prevent false positives for columns
that are only in some of the models sharing the same table.
Example of this is 'sale_id' not being a field of stock.picking.in
""" """
res = [] res = []
model_pool = self.pool['ir.model'] model_pool = self.pool['ir.model']
model_ids = model_pool.search(cr, uid, [], context=context) model_ids = model_pool.search(cr, uid, [], context=context)
line_pool = self.pool['cleanup.purge.line.column']
# mapping of tables to tuples (model id, [pool1, pool2, ...])
table2model = {}
for model in model_pool.browse(cr, uid, model_ids, context=context): for model in model_pool.browse(cr, uid, model_ids, context=context):
model_pool = self.pool.get(model.model) model_pool = self.pool.get(model.model)
if not model_pool or not model_pool._auto: if not model_pool or not model_pool._auto:
continue continue
table2model.setdefault(
model_pool._table, (model.id, []))[1].append(model_pool)
for table, model_spec in table2model.iteritems():
for column in self.get_orphaned_columns( for column in self.get_orphaned_columns(
cr, uid, model_pool, context=context):
cr, uid, model_spec[1], context=context):
res.append((0, 0, { res.append((0, 0, {
'name': column, 'name': column,
'model_id': model.id}))
'model_id': model_spec[0]}))
if not res: if not res:
raise orm.except_orm( raise orm.except_orm(
_('Nothing to do'), _('Nothing to do'),

106
database_cleanup/model/purge_data.py

@ -0,0 +1,106 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# This module copyright (C) 2014 Therp BV (<http://therp.nl>).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp.osv import orm, fields
from openerp.tools.translate import _
class CleanupPurgeLineData(orm.TransientModel):
_inherit = 'cleanup.purge.line'
_name = 'cleanup.purge.line.data'
_columns = {
'data_id': fields.many2one(
'ir.model.data', 'Data entry',
ondelete='SET NULL'),
'wizard_id': fields.many2one(
'cleanup.purge.wizard.data', 'Purge Wizard', readonly=True),
}
def purge(self, cr, uid, ids, context=None):
"""
Unlink data entries upon manual confirmation.
"""
data_ids = []
for line in self.browse(cr, uid, ids, context=context):
if line.purged or not line.data_id:
continue
data_ids.append(line.data_id.id)
self.logger.info('Purging data entry: %s', line.name)
self.pool['ir.model.data'].unlink(cr, uid, data_ids, context=context)
return self.write(cr, uid, ids, {'purged': True}, context=context)
class CleanupPurgeWizardData(orm.TransientModel):
_inherit = 'cleanup.purge.wizard'
_name = 'cleanup.purge.wizard.data'
def default_get(self, cr, uid, fields, context=None):
res = super(CleanupPurgeWizardData, self).default_get(
cr, uid, fields, context=context)
if 'name' in fields:
res['name'] = _('Purge data')
return res
def find(self, cr, uid, context=None):
"""
Collect all rows from ir_model_data that refer
to a nonexisting model, or to a nonexisting
row in the model's table.
"""
res = []
data_pool = self.pool['ir.model.data']
data_ids = []
unknown_models = []
cr.execute("""SELECT DISTINCT(model) FROM ir_model_data""")
for (model,) in cr.fetchall():
if not model:
continue
if not self.pool.get(model):
unknown_models.append(model)
continue
cr.execute(
"""
SELECT id FROM ir_model_data
WHERE model = %%s
AND res_id IS NOT NULL
AND res_id NOT IN (
SELECT id FROM %s)
""" % self.pool[model]._table, (model,))
data_ids += [data_row[0] for data_row in cr.fetchall()]
data_ids += data_pool.search(
cr, uid, [('model', 'in', unknown_models)], context=context)
for data in data_pool.browse(cr, uid, data_ids, context=context):
res.append((0, 0, {
'data_id': data.id,
'name': "%s.%s, object of type %s" % (
data.module, data.name, data.model)}))
if not res:
raise orm.except_orm(
_('Nothing to do'),
_('No orphaned data entries found'))
return res
_columns = {
'purge_line_ids': fields.one2many(
'cleanup.purge.line.data',
'wizard_id', 'Data to purge'),
}

9
database_cleanup/model/purge_models.py

@ -80,9 +80,14 @@ class CleanupPurgeLineModel(orm.TransientModel):
cr, uid, constraint_ids, context=context) cr, uid, constraint_ids, context=context)
relation_ids = fields_pool.search( relation_ids = fields_pool.search(
cr, uid, [('relation', '=', row[1])], context=context) cr, uid, [('relation', '=', row[1])], context=context)
if relation_ids:
fields_pool.unlink(cr, uid, relation_ids,
for relation in relation_ids:
try:
# Fails if the model on the target side
# cannot be instantiated
fields_pool.unlink(cr, uid, [relation],
context=local_context) context=local_context)
except AttributeError:
pass
model_pool.unlink(cr, uid, [row[0]], context=local_context) model_pool.unlink(cr, uid, [row[0]], context=local_context)
line.write({'purged': True}) line.write({'purged': True})
cr.commit() cr.commit()

11
database_cleanup/model/purge_tables.py

@ -56,8 +56,8 @@ class CleanupPurgeLineTable(orm.TransientModel):
FROM pg_attribute af, pg_attribute a, FROM pg_attribute af, pg_attribute a,
(SELECT conname, conrelid, confrelid,conkey[i] AS conkey, (SELECT conname, conrelid, confrelid,conkey[i] AS conkey,
confkey[i] AS confkey confkey[i] AS confkey
FROM (select conname, conrelid, confrelid, conkey, confkey,
generate_series(1,array_upper(conkey,1)) AS i
FROM (select conname, conrelid, confrelid, conkey,
confkey, generate_series(1,array_upper(conkey,1)) AS i
FROM pg_constraint WHERE contype = 'f') ss) ss2 FROM pg_constraint WHERE contype = 'f') ss) ss2
WHERE af.attnum = confkey AND af.attrelid = confrelid AND WHERE af.attnum = confkey AND af.attrelid = confrelid AND
a.attnum = conkey AND a.attrelid = conrelid a.attnum = conkey AND a.attrelid = conrelid
@ -80,6 +80,7 @@ class CleanupPurgeLineTable(orm.TransientModel):
cr.commit() cr.commit()
return True return True
class CleanupPurgeWizardTable(orm.TransientModel): class CleanupPurgeWizardTable(orm.TransientModel):
_inherit = 'cleanup.purge.wizard' _inherit = 'cleanup.purge.wizard'
_name = 'cleanup.purge.wizard.table' _name = 'cleanup.purge.wizard.table'
@ -88,7 +89,7 @@ class CleanupPurgeWizardTable(orm.TransientModel):
res = super(CleanupPurgeWizardTable, self).default_get( res = super(CleanupPurgeWizardTable, self).default_get(
cr, uid, fields, context=context) cr, uid, fields, context=context)
if 'name' in fields: if 'name' in fields:
res['name'] = _('Purge modules')
res['name'] = _('Purge tables')
return res return res
def find(self, cr, uid, context=None): def find(self, cr, uid, context=None):
@ -97,8 +98,8 @@ class CleanupPurgeWizardTable(orm.TransientModel):
Ignore views for now. Ignore views for now.
""" """
model_ids = self.pool['ir.model'].search(cr, uid, [], context=context) model_ids = self.pool['ir.model'].search(cr, uid, [], context=context)
line_pool = self.pool['cleanup.purge.line.table']
known_tables = []
# Start out with known tables with no model
known_tables = ['wkf_witm_trans']
for model in self.pool['ir.model'].browse( for model in self.pool['ir.model'].browse(
cr, uid, model_ids, context=context): cr, uid, model_ids, context=context):

1
database_cleanup/model/purge_wizard.py

@ -36,6 +36,7 @@ class CleanupPurgeLine(orm.AbstractModel):
def purge(self, cr, uid, ids, context=None): def purge(self, cr, uid, ids, context=None):
raise NotImplementedError raise NotImplementedError
class PurgeWizard(orm.AbstractModel): class PurgeWizard(orm.AbstractModel):
""" Abstract base class for the purge wizards """ """ Abstract base class for the purge wizards """
_name = 'cleanup.purge.wizard' _name = 'cleanup.purge.wizard'

9
database_cleanup/view/menu.xml

@ -32,10 +32,17 @@
<record model="ir.ui.menu" id="menu_purge_tables"> <record model="ir.ui.menu" id="menu_purge_tables">
<field name="name">Purge obsolete tables</field> <field name="name">Purge obsolete tables</field>
<field name="sequence" eval="30" />
<field name="sequence" eval="40" />
<field name="action" ref="action_purge_tables" /> <field name="action" ref="action_purge_tables" />
<field name="parent_id" ref="menu_database_cleanup"/> <field name="parent_id" ref="menu_database_cleanup"/>
</record> </record>
<record model="ir.ui.menu" id="menu_purge_data">
<field name="name">Purge obsolete data entries</field>
<field name="sequence" eval="50" />
<field name="action" ref="action_purge_data" />
<field name="parent_id" ref="menu_database_cleanup"/>
</record>
</data> </data>
</openerp> </openerp>

37
database_cleanup/view/purge_data.xml

@ -0,0 +1,37 @@
<?xml version="1.0" encoding="utf-8"?>
<openerp>
<data>
<record id="purge_data_view" model="ir.ui.view">
<field name="name">Form view for purge data wizard</field>
<field name="model">cleanup.purge.wizard.data</field>
<field name="arch" type="xml">
<form string="Purge data entries that refer to missing resources" version="7.0">
<h1>
<field name="name"/>
</h1>
<button type="object" name="purge_all" string="Purge all data" />
<field name="purge_line_ids" colspan="4" nolabel="1">
<tree string="Purge data">
<field name="name" />
<field name="data_id" />
<field name="purged" invisible="0" />
<button type="object" name="purge"
icon="gtk-cancel" string="Purge this data"
attrs="{'invisible': [('purged', '=', True)]}"/>
</tree>
</field>
</form>
</field>
</record>
<record id="action_purge_data" model="ir.actions.act_window">
<field name="name">Purge data entries that refer to missing resources</field>
<field name="type">ir.actions.act_window</field>
<field name="res_model">cleanup.purge.wizard.data</field>
<field name="view_type">form</field>
<field name="view_mode">form</field>
</record>
</data>
</openerp>
Loading…
Cancel
Save