[FIX] Don't remove uid field from wkf_instance, which is written in

raw SQL query (but never read afterwards). Workaround for
    lp:1277899

[FIX] Preserve dangling workflow table which is in use

[RFR] Group models per table when detecting columns to purge
      to prevent problems with models sharing the same table

[ADD] Allow purging of dangling data entries

[FIX] Data purging now working

[IMP] Docstrings

[FIX] Label
[FIX] Catch attempt to unlink field from nonexisting model

[RFR] Flake8

database_cleanup/__openerp__.py

@@ -31,12 +31,13 @@
         'view/purge_models.xml',
         'view/purge_columns.xml',
         'view/purge_tables.xml',
+        'view/purge_data.xml',
         'view/menu.xml',
         ],
     'description': """\
 Clean your OpenERP database from remnants of modules, models, columns and
-tables left by uninstalled modules (prior to 7.0) or a homebrew database upgrade
-to a new major version of OpenERP.
+tables left by uninstalled modules (prior to 7.0) or a homebrew database
+upgrade to a new major version of OpenERP.
 
 After installation of this module, go to the Settings menu -> Technical ->
 Database cleanup. Go through the modules, models, columns and tables

database_cleanup/model/__init__.py

@@ -3,3 +3,4 @@ from . import purge_modules
 from . import purge_models
 from . import purge_columns
 from . import purge_tables
+from . import purge_data

database_cleanup/model/purge_columns.py

@@ -53,7 +53,7 @@ class CleanupPurgeLineColumn(orm.TransientModel):
                 'WHERE attrelid = '
                 '( SELECT oid FROM pg_class WHERE relname = %s ) '
                 'AND attname = %s',
-                (model_pool._table, line.name));
+                (model_pool._table, line.name))
             if not cr.fetchone()[0]:
                 continue
 
@@ -68,10 +68,17 @@ class CleanupPurgeLineColumn(orm.TransientModel):
             cr.commit()
         return True
 
+
 class CleanupPurgeWizardColumn(orm.TransientModel):
     _inherit = 'cleanup.purge.wizard'
     _name = 'cleanup.purge.wizard.column'
 
+    # List of known columns in use without corresponding fields
+    # Format: {table: [fields]}
+    blacklist = {
+        'wkf_instance': ['uid'],  # lp:1277899
+        }
+
     def default_get(self, cr, uid, fields, context=None):
         res = super(CleanupPurgeWizardColumn, self).default_get(
             cr, uid, fields, context=context)
@@ -79,17 +86,22 @@ class CleanupPurgeWizardColumn(orm.TransientModel):
             res['name'] = _('Purge columns')
         return res
 
-    def get_orphaned_columns(self, cr, uid, model_pool, context=None):
+    def get_orphaned_columns(self, cr, uid, model_pools, context=None):
         """
         From openobject-server/openerp/osv/orm.py
         Iterate on the database columns to identify columns
         of fields which have been removed
         """
-        columns = [
-            c for c in model_pool._columns
-            if not (isinstance(model_pool._columns[c], fields.function)
-                    and not model_pool._columns[c].store)]
+
+        columns = list(set([
+            column for model_pool in model_pools
+            for column in model_pool._columns
+            if not (isinstance(model_pool._columns[column], fields.function)
+                    and not model_pool._columns[column].store)
+            ]))
         columns += orm.MAGIC_COLUMNS
+        columns += self.blacklist.get(model_pools[0]._table, [])
+
         cr.execute("SELECT a.attname"
                    "  FROM pg_class c, pg_attribute a"
                    " WHERE c.relname=%s"
@@ -98,26 +110,37 @@ class CleanupPurgeWizardColumn(orm.TransientModel):
                    "   AND pg_catalog.format_type(a.atttypid, a.atttypmod)"
                    "        NOT IN ('cid', 'tid', 'oid', 'xid')"
                    "   AND a.attname NOT IN %s",
-                   (model_pool._table, False, tuple(columns))),
+                   (model_pools[0]._table, False, tuple(columns))),
         return [column[0] for column in cr.fetchall()]
 
     def find(self, cr, uid, context=None):
         """
         Search for columns that are not in the corresponding model.
+
+        Group models by table to prevent false positives for columns
+        that are only in some of the models sharing the same table.
+        Example of this is 'sale_id' not being a field of stock.picking.in
         """
         res = []
         model_pool = self.pool['ir.model']
         model_ids = model_pool.search(cr, uid, [], context=context)
-        line_pool = self.pool['cleanup.purge.line.column']
+
+        # mapping of tables to tuples (model id, [pool1, pool2, ...])
+        table2model = {}
+
         for model in model_pool.browse(cr, uid, model_ids, context=context):
             model_pool = self.pool.get(model.model)
             if not model_pool or not model_pool._auto:
                 continue
+            table2model.setdefault(
+                model_pool._table, (model.id, []))[1].append(model_pool)
+
+        for table, model_spec in table2model.iteritems():
             for column in self.get_orphaned_columns(
-                    cr, uid, model_pool, context=context):
+                    cr, uid, model_spec[1], context=context):
                 res.append((0, 0, {
                             'name': column,
-                            'model_id': model.id}))
+                            'model_id': model_spec[0]}))
         if not res:
             raise orm.except_orm(
                 _('Nothing to do'),

database_cleanup/model/purge_data.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2014 Therp BV (<http://therp.nl>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+from openerp.osv import orm, fields
+from openerp.tools.translate import _
+
+
+class CleanupPurgeLineData(orm.TransientModel):
+    _inherit = 'cleanup.purge.line'
+    _name = 'cleanup.purge.line.data'
+
+    _columns = {
+        'data_id': fields.many2one(
+            'ir.model.data', 'Data entry',
+            ondelete='SET NULL'),
+        'wizard_id': fields.many2one(
+            'cleanup.purge.wizard.data', 'Purge Wizard', readonly=True),
+        }
+
+    def purge(self, cr, uid, ids, context=None):
+        """
+        Unlink data entries upon manual confirmation.
+        """
+        data_ids = []
+        for line in self.browse(cr, uid, ids, context=context):
+            if line.purged or not line.data_id:
+                continue
+            data_ids.append(line.data_id.id)
+            self.logger.info('Purging data entry: %s', line.name)
+        self.pool['ir.model.data'].unlink(cr, uid, data_ids, context=context)
+        return self.write(cr, uid, ids, {'purged': True}, context=context)
+
+
+class CleanupPurgeWizardData(orm.TransientModel):
+    _inherit = 'cleanup.purge.wizard'
+    _name = 'cleanup.purge.wizard.data'
+
+    def default_get(self, cr, uid, fields, context=None):
+        res = super(CleanupPurgeWizardData, self).default_get(
+            cr, uid, fields, context=context)
+        if 'name' in fields:
+            res['name'] = _('Purge data')
+        return res
+
+    def find(self, cr, uid, context=None):
+        """
+        Collect all rows from ir_model_data that refer
+        to a nonexisting model, or to a nonexisting
+        row in the model's table.
+        """
+        res = []
+        data_pool = self.pool['ir.model.data']
+        data_ids = []
+        unknown_models = []
+        cr.execute("""SELECT DISTINCT(model) FROM ir_model_data""")
+        for (model,) in cr.fetchall():
+            if not model:
+                continue
+            if not self.pool.get(model):
+                unknown_models.append(model)
+                continue
+            cr.execute(
+                """
+                SELECT id FROM ir_model_data
+                WHERE model = %%s
+                AND res_id IS NOT NULL
+                AND res_id NOT IN (
+                    SELECT id FROM %s)
+                """ % self.pool[model]._table, (model,))
+            data_ids += [data_row[0] for data_row in cr.fetchall()]
+        data_ids += data_pool.search(
+            cr, uid, [('model', 'in', unknown_models)], context=context)
+        for data in data_pool.browse(cr, uid, data_ids, context=context):
+            res.append((0, 0, {
+                        'data_id': data.id,
+                        'name': "%s.%s, object of type %s" % (
+                            data.module, data.name, data.model)}))
+        if not res:
+            raise orm.except_orm(
+                _('Nothing to do'),
+                _('No orphaned data entries found'))
+        return res
+
+    _columns = {
+        'purge_line_ids': fields.one2many(
+            'cleanup.purge.line.data',
+            'wizard_id', 'Data to purge'),
+        }

database_cleanup/model/purge_models.py

@@ -80,9 +80,14 @@ class CleanupPurgeLineModel(orm.TransientModel):
                         cr, uid, constraint_ids, context=context)
                 relation_ids = fields_pool.search(
                     cr, uid, [('relation', '=', row[1])], context=context)
-                if relation_ids:
-                    fields_pool.unlink(cr, uid, relation_ids,
+                for relation in relation_ids:
+                    try:
+                        # Fails if the model on the target side
+                        # cannot be instantiated
+                        fields_pool.unlink(cr, uid, [relation],
                                            context=local_context)
+                    except AttributeError:
+                        pass
                 model_pool.unlink(cr, uid, [row[0]], context=local_context)
                 line.write({'purged': True})
                 cr.commit()

database_cleanup/model/purge_tables.py

@@ -56,8 +56,8 @@ class CleanupPurgeLineTable(orm.TransientModel):
                 FROM pg_attribute af, pg_attribute a,
                     (SELECT conname, conrelid, confrelid,conkey[i] AS conkey,
                          confkey[i] AS confkey
-                     FROM (select conname, conrelid, confrelid, conkey, confkey,
-                       generate_series(1,array_upper(conkey,1)) AS i
+                     FROM (select conname, conrelid, confrelid, conkey,
+                       confkey, generate_series(1,array_upper(conkey,1)) AS i
                        FROM pg_constraint WHERE contype = 'f') ss) ss2
                 WHERE af.attnum = confkey AND af.attrelid = confrelid AND
                 a.attnum = conkey AND a.attrelid = conrelid
@@ -80,6 +80,7 @@ class CleanupPurgeLineTable(orm.TransientModel):
             cr.commit()
         return True
 
+
 class CleanupPurgeWizardTable(orm.TransientModel):
     _inherit = 'cleanup.purge.wizard'
     _name = 'cleanup.purge.wizard.table'
@@ -88,7 +89,7 @@ class CleanupPurgeWizardTable(orm.TransientModel):
         res = super(CleanupPurgeWizardTable, self).default_get(
             cr, uid, fields, context=context)
         if 'name' in fields:
-            res['name'] = _('Purge modules')
+            res['name'] = _('Purge tables')
         return res
 
     def find(self, cr, uid, context=None):
@@ -97,8 +98,8 @@ class CleanupPurgeWizardTable(orm.TransientModel):
         Ignore views for now.
         """
         model_ids = self.pool['ir.model'].search(cr, uid, [], context=context)
-        line_pool = self.pool['cleanup.purge.line.table']
-        known_tables = []
+        # Start out with known tables with no model
+        known_tables = ['wkf_witm_trans']
         for model in self.pool['ir.model'].browse(
                 cr, uid, model_ids, context=context):
 

database_cleanup/model/purge_wizard.py

@@ -36,6 +36,7 @@ class CleanupPurgeLine(orm.AbstractModel):
     def purge(self, cr, uid, ids, context=None):
         raise NotImplementedError
 
+
 class PurgeWizard(orm.AbstractModel):
     """ Abstract base class for the purge wizards """
     _name = 'cleanup.purge.wizard'

database_cleanup/view/menu.xml

@@ -32,10 +32,17 @@
 
         <record model="ir.ui.menu" id="menu_purge_tables">
             <field name="name">Purge obsolete tables</field>
-            <field name="sequence" eval="30" />
+            <field name="sequence" eval="40" />
             <field name="action" ref="action_purge_tables" />
             <field name="parent_id" ref="menu_database_cleanup"/>
         </record>
 
+        <record model="ir.ui.menu" id="menu_purge_data">
+            <field name="name">Purge obsolete data entries</field>
+            <field name="sequence" eval="50" />
+            <field name="action" ref="action_purge_data" />
+            <field name="parent_id" ref="menu_database_cleanup"/>
+        </record>
+
     </data>
 </openerp>

database_cleanup/view/purge_data.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
+<openerp>
+    <data>
+
+        <record id="purge_data_view" model="ir.ui.view">
+            <field name="name">Form view for purge data wizard</field>
+            <field name="model">cleanup.purge.wizard.data</field>
+            <field name="arch" type="xml">
+                <form string="Purge data entries that refer to missing resources" version="7.0">
+                    <h1>
+                        <field name="name"/>
+                    </h1>
+                    <button type="object" name="purge_all" string="Purge all data" />
+                    <field name="purge_line_ids" colspan="4" nolabel="1">
+                        <tree string="Purge data">
+                            <field name="name" />
+                            <field name="data_id" />
+                            <field name="purged" invisible="0" />
+                            <button type="object" name="purge"
+                                    icon="gtk-cancel" string="Purge this data"
+                                    attrs="{'invisible': [('purged', '=', True)]}"/>
+                        </tree>
+                    </field>
+                </form>
+            </field>
+        </record>
+
+        <record id="action_purge_data" model="ir.actions.act_window">
+            <field name="name">Purge data entries that refer to missing resources</field>
+            <field name="type">ir.actions.act_window</field>
+            <field name="res_model">cleanup.purge.wizard.data</field>
+            <field name="view_type">form</field>
+            <field name="view_mode">form</field>
+        </record>
+
+    </data>
+</openerp>