[IMP] Renamed module base_trgm_search to base_search_fuzzy, added Unit tests, added translations, added access permissions, moved the monkey patching to method _register_hook of ir.model and fixed _auto_init, added README, cleaned up some aprts

8 years ago · c4d5d0b7c6
15 changed files with 499 additions and 83 deletions
--- a/base_search_fuzzy/README.rst
+++ b/base_search_fuzzy/README.rst
@ -0,0 +1,99 @@
+.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+   :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+   :alt: License: AGPL-3
+
+=========================
+PostgreSQL Trigram Search
+=========================
+
+This addon provides the ability to create GIN or GiST indexes of char and text
+fields and also to use the search operator `%` in search domains. Currently
+this module doesn't change the backend search or something else. It provides
+only the possibilty to do a fuzzy search for external addons.
+
+
+Installation
+============
+
+First you need to have the ``pg_trgm`` extension available. In debian based
+distribution you have to install the `postgresql-contrib` module. Then you have
+to either install the ``pg_trgm`` extension to your database or you have to give
+your postgresql user the superadmin right (this allows the odoo module to
+install the extension to the database).
+
+
+Configuration
+=============
+
+If you installed the odoo module you can define ``GIN`` and ``GiST`` indexes for
+`char` and `text` via `Settings -> Database Structure -> Trigram Index`. The
+index name will automatically created for new entries.
+
+
+Usage
+=====
+
+For example you can create an index for the `name` field of `res.partner`. Then
+in a search you can use
+
+``self.env['res.partner'].search([('name', '%', 'Jon Miller)])``
+
+In this Example it can find existing names like `John Miller` or `John Mill`.
+
+Which strings can be found depends on the limit which is set (default: 0.3).
+Currently you have to set the limit by executing SQL:
+
+``self.env.cr.execute("SELECT set_limit(0.2);")``
+
+Also you can use the ``similarity(column, 'text')`` function in the ``order``
+parameter to order by the similarity. This is just a basic implementation which
+doesn't contains validations and has to start with this function. For example
+you can define this like:
+
+``similarity(%s.name, 'John Mil') DESC" % self.env['res.partner']._table``
+
+For further questions read the Documentation of the
+`pg_trgm <https://www.postgresql.org/docs/current/static/pgtrgm.html>`_ module.
+
+Known issues / Roadmap
+======================
+
+* Modify the general search parts (e.g. in tree view or many2one fields)
+* add better `order by` handling
+
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues
+<https://github.com/OCA/server-tools/issues>`_. In case of trouble, please
+check there if your issue has already been reported. If you spotted it first,
+help us smashing it by providing a detailed and welcomed feedback.
+
+Credits
+=======
+
+Images
+------
+
+* Odoo Community Association: `Icon <https://github.com/OCA/maintainer-tools/blob/master/template/module/static/description/icon.svg>`_.
+
+Contributors
+------------
+
+* Christoph Giesel <https://github.com/christophlsa>
+
+Maintainer
+----------
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+This module is maintained by the OCA.
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+To contribute to this module, please visit https://odoo-community.org.
--- a/base_search_fuzzy/init.py
+++ b/base_search_fuzzy/init.py
@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+from . import models
--- a/base_search_fuzzy/openerp.py
+++ b/base_search_fuzzy/openerp.py
@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+{
+    'name': "Fuzzy Search",
+    'summary': "Fuzzy search with the PostgreSQL trigram extension",
+    'category': 'Uncategorized',
+    'version': '8.0.1.0.0',
+    'website': 'https://odoo-community.org/',
+    'author': 'bloopark systems GmbH & Co. KG, '
+              'Odoo Community Association (OCA)',
+    'license': 'AGPL-3',
+    'depends': [
+        'base',
+    ],
+    'data': [
+        'views/trgm_index.xml',
+        'security/ir.model.access.csv',
+    ],
+    'installable': True,
+}
--- a/base_search_fuzzy/i18n/base_search_fuzzy.pot
+++ b/base_search_fuzzy/i18n/base_search_fuzzy.pot
@ -0,0 +1,94 @@
+# Translation of Odoo Server.
+# This file contains the translation of the following modules:
+#	* base_search_fuzzy
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: Odoo Server 9.0alpha1\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2016-06-24 08:47+0000\n"
+"PO-Revision-Date: 2016-06-24 08:47+0000\n"
+"Last-Translator: <>\n"
+"Language-Team: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: \n"
+"Plural-Forms: \n"
+
+#. module: base_search_fuzzy
+#: help:trgm.index,index_type:0
+msgid "Cite from PostgreSQL documentation: \"As a rule of thumb, a GIN index is faster to search than a GiST index, but slower to build or update; so GIN is better suited for static data and GiST for often-updated data.\""
+msgstr ""
+
+#. module: base_search_fuzzy
+#: field:trgm.index,create_uid:0
+msgid "Created by"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: field:trgm.index,create_date:0
+msgid "Created on"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: field:trgm.index,field_id:0
+msgid "Field"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: selection:trgm.index,index_type:0
+msgid "GIN"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: selection:trgm.index,index_type:0
+msgid "GiST"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: field:trgm.index,id:0
+msgid "ID"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: field:trgm.index,index_name:0
+msgid "Index Name"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: field:trgm.index,index_type:0
+msgid "Index Type"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: field:trgm.index,write_uid:0
+msgid "Last Updated by"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: field:trgm.index,write_date:0
+msgid "Last Updated on"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: model:ir.model,name:base_search_fuzzy.model_ir_model
+msgid "Models"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: help:trgm.index,index_name:0
+msgid "The index name is automatically generated like fieldname_indextype_idx. If the index already exists and the index is located in the same table then this index is resused. If the index is located in another table then a number is added at the end of the index name."
+msgstr ""
+
+#. module: base_search_fuzzy
+#: model:ir.actions.act_window,name:base_search_fuzzy.trgm_index_action
+#: model:ir.ui.menu,name:base_search_fuzzy.trgm_index_menu
+#: view:trgm.index:base_search_fuzzy.trgm_index_view_form
+#: view:trgm.index:base_search_fuzzy.trgm_index_view_tree
+msgid "Trigram Index"
+msgstr ""
+
+#. module: base_search_fuzzy
+#: help:trgm.index,field_id:0
+msgid "You can either select a field of type \"text\" or \"char\"."
+msgstr ""
--- a/base_search_fuzzy/i18n/de.po
+++ b/base_search_fuzzy/i18n/de.po
@ -0,0 +1,94 @@
+# Translation of Odoo Server.
+# This file contains the translation of the following modules:
+#	* base_search_fuzzy
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: Odoo Server 9.0alpha1\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2016-06-24 08:49+0000\n"
+"PO-Revision-Date: 2016-06-24 08:49+0000\n"
+"Last-Translator: <>\n"
+"Language-Team: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: \n"
+"Plural-Forms: \n"
+
+#. module: base_search_fuzzy
+#: help:trgm.index,index_type:0
+msgid "Cite from PostgreSQL documentation: \"As a rule of thumb, a GIN index is faster to search than a GiST index, but slower to build or update; so GIN is better suited for static data and GiST for often-updated data.\""
+msgstr "Zitat aus der PostgreSQL Dokumentation: \"Eine Fausregel ist, ein GIN Index ist schneller durchzusuchen als ein GiST Index, aber langsamer aufzubauen und zu aktualisieren; so ist GIN besser geeignet für statische Daten und GiST für oft aktualisierte Daten.\""
+
+#. module: base_search_fuzzy
+#: field:trgm.index,create_uid:0
+msgid "Created by"
+msgstr "Erstellt durch"
+
+#. module: base_search_fuzzy
+#: field:trgm.index,create_date:0
+msgid "Created on"
+msgstr "Erstellt am"
+
+#. module: base_search_fuzzy
+#: field:trgm.index,field_id:0
+msgid "Field"
+msgstr "Feld"
+
+#. module: base_search_fuzzy
+#: selection:trgm.index,index_type:0
+msgid "GIN"
+msgstr "GIN"
+
+#. module: base_search_fuzzy
+#: selection:trgm.index,index_type:0
+msgid "GiST"
+msgstr "GiST"
+
+#. module: base_search_fuzzy
+#: field:trgm.index,id:0
+msgid "ID"
+msgstr "ID"
+
+#. module: base_search_fuzzy
+#: field:trgm.index,index_name:0
+msgid "Index Name"
+msgstr "Index Name"
+
+#. module: base_search_fuzzy
+#: field:trgm.index,index_type:0
+msgid "Index Type"
+msgstr "Index Typ"
+
+#. module: base_search_fuzzy
+#: field:trgm.index,write_uid:0
+msgid "Last Updated by"
+msgstr "Zuletzt aktualisiert durch"
+
+#. module: base_search_fuzzy
+#: field:trgm.index,write_date:0
+msgid "Last Updated on"
+msgstr "Zuletzt aktualisiert am"
+
+#. module: base_search_fuzzy
+#: model:ir.model,name:base_search_fuzzy.model_ir_model
+msgid "Models"
+msgstr "Datenmodelle"
+
+#. module: base_search_fuzzy
+#: help:trgm.index,index_name:0
+msgid "The index name is automatically generated like fieldname_indextype_idx. If the index already exists and the index is located in the same table then this index is resused. If the index is located in another table then a number is added at the end of the index name."
+msgstr "Der Index Name wird automatisch im Format feldname_indextyp_idx generiert. Falls der Index bereits existiert und der Index sich in der selben Tabelle befindet, dann wird dieser Index wiederverwendet. Falls der Index in einer anderen Tabelle existiert, dann wird eine Zahl an das Ende des Index Namens angefügt."
+
+#. module: base_search_fuzzy
+#: model:ir.actions.act_window,name:base_search_fuzzy.trgm_index_action
+#: model:ir.ui.menu,name:base_search_fuzzy.trgm_index_menu
+#: view:trgm.index:base_search_fuzzy.trgm_index_view_form
+#: view:trgm.index:base_search_fuzzy.trgm_index_view_tree
+msgid "Trigram Index"
+msgstr "Trigram Index"
+
+#. module: base_search_fuzzy
+#: help:trgm.index,field_id:0
+msgid "You can either select a field of type \"text\" or \"char\"."
+msgstr "Sie können entweder Felder vom Typ \"text\" oder \"char\" auswählen."
--- a/base_search_fuzzy/models/init.py
+++ b/base_search_fuzzy/models/init.py
@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+from . import ir_model
+from . import trgm_index
--- a/base_search_fuzzy/models/ir_model.py
+++ b/base_search_fuzzy/models/ir_model.py
@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Odoo, an open source suite of business apps
+# This module copyright (C) 2015 bloopark systems (<http://bloopark.de>).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import logging
+
+from openerp import models
+from openerp.osv import expression
+
+
+_logger = logging.getLogger(__name__)
+
+
+def patch_leaf_trgm(method):
+    def decorate_leaf_to_sql(self, eleaf):
+        model = eleaf.model
+        leaf = eleaf.leaf
+        left, operator, right = leaf
+        table_alias = '"%s"' % (eleaf.generate_alias())
+
+        if operator == '%':
+            sql_operator = '%%'
+            params = []
+
+            if left in model._columns:
+                format = model._columns[left]._symbol_set[0]
+                column = '%s.%s' % (table_alias, expression._quote(left))
+                query = '(%s %s %s)' % (column, sql_operator, format)
+            elif left in expression.MAGIC_COLUMNS:
+                query = "(%s.\"%s\" %s %%s)" % (
+                    table_alias, left, sql_operator)
+                params = right
+            else:  # Must not happen
+                raise ValueError(
+                    "Invalid field %r in domain term %r" % (left, leaf))
+
+            if left in model._columns:
+                params = model._columns[left]._symbol_set[1](right)
+
+            if isinstance(params, basestring):
+                params = [params]
+            return query, params
+        elif operator == 'inselect':
+            right = (right[0].replace(' % ', ' %% '), right[1])
+            eleaf.leaf = (left, operator, right)
+        return method(self, eleaf)
+
+    return decorate_leaf_to_sql
+
+
+def patch_generate_order_by(method):
+    def decorate_generate_order_by(self, order_spec, query):
+        if order_spec and order_spec.startswith('similarity('):
+            return ' ORDER BY ' + order_spec
+        return method(self, order_spec, query)
+
+    return decorate_generate_order_by
+
+
+class IrModel(models.Model):
+
+    _inherit = 'ir.model'
+
+    def _register_hook(self, cr, ids=None):
+        expression.expression._expression__leaf_to_sql = patch_leaf_trgm(
+            expression.expression._expression__leaf_to_sql)
+
+        expression.TERM_OPERATORS += ('%',)
+
+        models.BaseModel._generate_order_by = patch_generate_order_by(
+            models.BaseModel._generate_order_by)
+
+        return super(IrModel, self)._register_hook(cr)
--- a/base_search_fuzzy/models/trgm_index.py
+++ b/base_search_fuzzy/models/trgm_index.py
@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
 import logging

-from openerp import api, exceptions, fields, models
-from openerp.osv import expression
+from openerp import SUPERUSER_ID, api, exceptions, fields, models

 from psycopg2.extensions import AsIs

@ -86,10 +86,11 @@ class TrgmIndex(models.Model):
            return True
        return False

-    @api.model
-    def _auto_init(self):
-        res = super(TrgmIndex, self)._auto_init()
-        self._install_trgm_extension()
+    def _auto_init(self, cr, context=None):
+        res = super(TrgmIndex, self)._auto_init(cr, context)
+        if self._install_trgm_extension(cr, SUPERUSER_ID, context=context):
+            _logger.info('The pg_trgm is loaded in the database and the '
+                         'fuzzy search can be used.')
        return res

    @api.model
@ -168,57 +169,3 @@ class TrgmIndex(models.Model):
                'index': AsIs(rec.index_name),
            })
        return super(TrgmIndex, self).unlink()
-
-
-def patch_leaf_trgm(method):
-    def decorate_leaf_to_sql(self, eleaf):
-        model = eleaf.model
-        leaf = eleaf.leaf
-        left, operator, right = leaf
-        table_alias = '"%s"' % (eleaf.generate_alias())
-
-        if operator == '%':
-            sql_operator = '%%'
-            params = []
-
-            if left in model._columns:
-                format = model._columns[left]._symbol_set[0]
-                column = '%s.%s' % (table_alias, expression._quote(left))
-                query = '(%s %s %s)' % (column, sql_operator, format)
-            elif left in expression.MAGIC_COLUMNS:
-                query = "(%s.\"%s\" %s %%s)" % (
-                    table_alias, left, sql_operator)
-                params = right
-            else:  # Must not happen
-                raise ValueError(
-                    "Invalid field %r in domain term %r" % (left, leaf))
-
-            if left in model._columns:
-                params = model._columns[left]._symbol_set[1](right)
-
-            if isinstance(params, basestring):
-                params = [params]
-            return query, params
-        elif operator == 'inselect':
-            right = (right[0].replace(' % ', ' %% '), right[1])
-            eleaf.leaf = (left, operator, right)
-        return method(self, eleaf)
-    return decorate_leaf_to_sql
-
-expression.expression._expression__leaf_to_sql = patch_leaf_trgm(
-    expression.expression._expression__leaf_to_sql)
-
-expression.TERM_OPERATORS += ('%',)
-
-
-def patch_generate_order_by(method):
-    def decorate_generate_order_by(self, order_spec, query):
-        if order_spec and order_spec.startswith('similarity('):
-            from pprint import pprint
-            pprint(query)
-            return ' ORDER BY ' + order_spec
-        return method(self, order_spec, query)
-    return decorate_generate_order_by
-
-models.BaseModel._generate_order_by = patch_generate_order_by(
-    models.BaseModel._generate_order_by)
--- a/base_search_fuzzy/security/ir.model.access.csv
+++ b/base_search_fuzzy/security/ir.model.access.csv
@ -0,0 +1,5 @@
+id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
+access_trgm_index_public,trgm_index group_public,model_trgm_index,base.group_public,1,0,0,0
+access_trgm_index_portal,trgm_index group_portal,model_trgm_index,base.group_portal,1,0,0,0
+access_trgm_index_group_partner_manager,trgm_index group_partner_manager,model_trgm_index,base.group_no_one,1,1,1,1
+access_trgm_index_group_user,trgm_index group_user,model_trgm_index,base.group_user,1,0,0,0
--- a/base_search_fuzzy/tests/init.py
+++ b/base_search_fuzzy/tests/init.py
@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+from . import test_query_generation
--- a/base_search_fuzzy/tests/test_query_generation.py
+++ b/base_search_fuzzy/tests/test_query_generation.py
@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+from openerp.osv import expression
+from openerp.tests.common import TransactionCase, at_install, post_install
+
+
+@at_install(False)
+@post_install(True)
+class QueryGenerationCase(TransactionCase):
+
+    def setUp(self):
+        super(QueryGenerationCase, self).setUp()
+        self.ResPartner = self.env['res.partner']
+        self.TrgmIndex = self.env['trgm.index']
+
+    def test_fuzzy_where_generation(self):
+        """Check the generation of the where clause."""
+        # the added fuzzy search operator should be available in the allowed
+        # operators
+        self.assertIn('%', expression.TERM_OPERATORS)
+
+        # create new query with fuzzy search operator
+        query = self.ResPartner._where_calc(
+            [('name', '%', 'test')], active_test=False)
+        from_clause, where_clause, where_clause_params = query.get_sql()
+
+        # the % parameter has to be escaped (%%) for the string replation
+        self.assertEqual(where_clause, """("res_partner"."name" %% %s)""")
+
+        # test the right sql query statement creation
+        # now there should be only one '%'
+        complete_where = self.env.cr.mogrify(
+            "SELECT FROM %s WHERE %s" % (from_clause, where_clause),
+            where_clause_params)
+        self.assertEqual(
+            complete_where,
+            'SELECT FROM "res_partner" WHERE '
+            '("res_partner"."name" % \'test\')')
+
+    def test_fuzzy_order_generation(self):
+        """Check the generation of the where clause."""
+        order = "similarity(%s.name, 'test') DESC" % self.ResPartner._table
+        query = self.ResPartner._where_calc(
+            [('name', '%', 'test')], active_test=False)
+        order_by = self.ResPartner._generate_order_by(order, query)
+        self.assertEqual(' ORDER BY %s' % order, order_by)
+
+    def test_fuzzy_search(self):
+        """Test the fuzzy search itself."""
+        if self.TrgmIndex._trgm_extension_exists() != 'installed':
+            return
+
+        if not self.TrgmIndex.index_exists('res.partner', 'name'):
+            field_partner_name = self.env.ref('base.field_res_partner_name')
+            self.TrgmIndex.create({
+                'field_id': field_partner_name.id,
+                'index_type': 'gin',
+            })
+
+        partner1 = self.ResPartner.create({
+            'name': 'John Smith'
+        })
+        partner2 = self.ResPartner.create(
+            {'name': 'John Smizz'}
+        )
+        partner3 = self.ResPartner.create({
+            'name': 'Linus Torvalds'
+        })
+
+        res = self.ResPartner.search([('name', '%', 'Jon Smith')])
+        self.assertIn(partner1.id, res.ids)
+        self.assertIn(partner2.id, res.ids)
+        self.assertNotIn(partner3.id, res.ids)
+
+        res = self.ResPartner.search([('name', '%', 'Smith John')])
+        self.assertIn(partner1.id, res.ids)
+        self.assertIn(partner2.id, res.ids)
+        self.assertNotIn(partner3.id, res.ids)
--- a/base_search_fuzzy/views/trgm_index.xml
+++ b/base_search_fuzzy/views/trgm_index.xml
@ -31,7 +31,7 @@
    </record>

    <record model="ir.actions.act_window" id="trgm_index_action">
-      <field name="name">Trigam Index</field>
+      <field name="name">Trigram Index</field>
      <field name="res_model">trgm.index</field>
      <field name="view_type">form</field>
      <field name="view_mode">tree,form</field>
@ -40,7 +40,8 @@

    <menuitem id="trgm_index_menu"
              parent="base.next_id_9"
-              action="trgm_index_action"/>
+              action="trgm_index_action"
+              groups="base.group_no_one"/>

  </data>
 </openerp>
--- a/base_trgm_search/init.py
+++ b/base_trgm_search/init.py
@ -1,2 +0,0 @@
-# -*- coding: utf-8 -*-
-from . import models
--- a/base_trgm_search/openerp.py
+++ b/base_trgm_search/openerp.py
@ -1,17 +0,0 @@
-# -*- coding: utf-8 -*-
-{
-    'name': "PostgreSQL Trigram Search",
-    'summary': """PostgreSQL Trigram Search""",
-    'category': 'Uncategorized',
-    'version': '8.0.1.0.0',
-    'website': 'https://odoo-community.org/',
-    'author': 'Daniel Reis, Odoo Community Association (OCA)',
-    'license': 'AGPL-3',
-    'depends': [
-        'base',
-    ],
-    'data': [
-        'views/trgm_index.xml',
-    ],
-    'installable': True,
-}
--- a/base_trgm_search/models/init.py
+++ b/base_trgm_search/models/init.py
@ -1,2 +0,0 @@
-# -*- coding: utf-8 -*-
-from . import trgm_index