[REF] remove use of cloc. use pygount librairy instead

.travis.yml

@@ -10,7 +10,6 @@ addons:
   apt:
     packages:
       - expect-dev  # provides unbuffer utility
-      - cloc  # module_analysis
 
 env:
   global:

module_analysis/__manifest__.py

@@ -27,7 +27,7 @@
         'data/ir_module_type_rule.xml',
     ],
     'external_dependencies': {
-        'lib': ['cloc'],
+        'python': ['pygount'],
     },
     'post_init_hook': 'analyse_installed_modules',
     'installable': True,

module_analysis/models/ir_module_module.py

@@ -2,10 +2,13 @@
 # @author: Sylvain LE GAL (https://twitter.com/legalsylvain)
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 
-import json
-import logging
 import os
-import subprocess
+# import json
+import pygount
+from pathlib import Path
+import logging
+# import os
+# import subprocess
 
 from odoo import api, fields, models
 from odoo.tools.safe_eval import safe_eval
@@ -24,14 +27,56 @@ class IrModuleModule(models.Model):
     module_type_id = fields.Many2one(
         string='Module Type', comodel_name='ir.module.type', readonly=True)
 
-    python_lines_qty = fields.Integer(string='Python Lines', readonly=True)
+    python_code_qty = fields.Integer(string='Python Lines', readonly=True)
 
-    xml_lines_qty = fields.Integer(string='XML Lines', readonly=True)
+    xml_code_qty = fields.Integer(string='XML Lines', readonly=True)
 
-    js_lines_qty = fields.Integer(string='JS Lines', readonly=True)
+    js_code_qty = fields.Integer(string='JS Lines', readonly=True)
 
-    css_lines_qty = fields.Integer(string='CSS Lines', readonly=True)
+    css_code_qty = fields.Integer(string='CSS Lines', readonly=True)
 
+    # Overloadable Section
+    @api.model
+    def _get_analyse_settings(self):
+        """Return a dictionnary of data analysed
+        Overload this function if you want to analyse other data
+        {
+            'extension': {
+                'data_to_analyse': 'field_name',
+            },
+        }, [...]
+        extension: extension of the file, with the '.'
+        data_to_analyse : possible value : code, documentation, empty, string
+        field_name: Odoo field name to store the analysis
+        """
+        return {
+            '.py': {
+                'code': 'python_code_qty',
+            },
+            '.xml': {
+                'code': 'xml_code_qty',
+            },
+            '.js': {
+                'code': 'js_code_qty',
+            },
+            '.css': {
+                'code': 'css_code_qty',
+            },
+        }
+
+    @api.model
+    def _get_clean_analyse_values(self):
+        """List of fields to unset when a module is uninstalled"""
+        return {
+            'author_ids': [6, 0, []],
+            'module_type_id': False,
+            'python_code_qty': False,
+            'xml_code_qty': 0,
+            'js_code_qty': 0,
+            'css_code_qty': 0,
+        }
+
+    # Overload Section
     @api.model
     def update_list(self):
         res = super(IrModuleModule, self).update_list()
@@ -46,24 +91,18 @@ class IrModuleModule(models.Model):
             self.button_analyze_code()
         elif vals.get('state', False) == 'uninstalled'\
                 and 'module_analysis' not in [x.name for x in self]:
-            self.write({
-                'author_ids': [6, 0, []],
-                'module_type_id': False,
-                'python_lines_qty': False,
-                'xml_lines_qty': 0,
-                'js_lines_qty': 0,
-                'css_lines_qty': 0,
-            })
+            self.write(self._get_clean_analyse_values())
         return res
 
+    # Public Section
     @api.multi
     def button_analyze_code(self):
         IrModuleAuthor = self.env['ir.module.author']
         IrModuleTypeRule = self.env['ir.module.type.rule']
         rules = IrModuleTypeRule.search([])
 
-        exclude_dir = "lib,description,tests,demo"
-        include_lang = self._get_analyzed_languages()
+        exclude_directories = ['description', 'lib', 'tests']
+        exclude_files = ['__openerp__.py', '__manifest__.py']
 
         for module in self:
             _logger.info("Analyzing Code for module %s" % (module.name))
@@ -88,40 +127,50 @@ class IrModuleModule(models.Model):
             module.module_type_id = module_type_id
 
             # Get Path of module folder and parse the code
-            path = get_module_path(module.name)
-
-            try:
-                command = [
-                    'cloc',
-                    '--exclude-dir=%s' % (exclude_dir),
-                    '--skip-uniqueness',
-                    '--include-lang=%s' % (include_lang),
-                    '--not-match-f="__openerp__.py|__manifest__.py"',
-                    '--json',
-                    os.path.join(path)]
-                temp = subprocess.Popen(command, stdout=subprocess.PIPE)
-                bytes_output = temp.communicate()
-                output = bytes_output[0].decode("utf-8").replace('\n', '')
-                json_res = json.loads(output)
-                values = self._prepare_values_from_json(json_res)
+            module_path = get_module_path(module.name)
+
+            # Get Files
+            analysed_datas = self._get_analyse_data_dict()
+            file_extensions = analysed_datas.keys()
+            file_list = self._get_files_to_analyse(
+                module_path, file_extensions, exclude_directories,
+                exclude_files)
+
+            for file_path, file_ext in file_list:
+                file_res = pygount.source_analysis(file_path, '')
+                for k, v in analysed_datas.get(file_ext).items():
+                    v['value'] += getattr(file_res, k)
+
+            # Update the module with the datas
+            values = {}
+            for file_ext, analyses in analysed_datas.items():
+                for k, v in analyses.items():
+                    values[v['field']] = v['value']
             module.write(values)
 
-            except Exception as e:
-                _logger.warning(
-                    'Failed to execute the cloc command on module %s\n'
-                    'Exception occured: %s' % (
-                        module.name, e))
-
+    # Custom Section
     @api.model
-    def _get_analyzed_languages(self):
-        "Overload the function to add extra languages to analyze"
-        return "Python,XML,JavaScript,CSS"
+    def _get_files_to_analyse(
+            self, path, file_extensions, exclude_directories, exclude_files):
+        res = []
+        for root, dirs, files in os.walk(path, followlinks=True):
+            if set(Path(root).parts) & set(exclude_directories):
+                continue
+            for name in files:
+                if name in exclude_files:
+                    continue
+                filename, file_extension = os.path.splitext(name)
+                if file_extension in file_extensions:
+                    res.append((os.path.join(root, name), file_extension))
+        return res
 
     @api.model
-    def _prepare_values_from_json(self, json_value):
-        return {
-            'python_lines_qty': json_value.get('Python', {}).get('code', 0),
-            'xml_lines_qty': json_value.get('XML', {}).get('code', 0),
-            'js_lines_qty': json_value.get('JavaScript', {}).get('code', 0),
-            'css_lines_qty': json_value.get('CSS', {}).get('code', 0),
+    def _get_analyse_data_dict(self):
+        res_dict = self._get_analyse_settings().copy()
+        for file_ext, analyse_dict in res_dict.items():
+            for analyse_type, v in analyse_dict.items():
+                analyse_dict[analyse_type] = {
+                    'field': v,
+                    'value': 0
                 }
+        return res_dict

module_analysis/readme/CONFIGURE.rst

@@ -23,3 +23,28 @@ to update the data, you have to :
     .. image:: ../static/description/base_module_update.png
 
 This will update analysis of your installed modules.
+
+
+Adding Extra data
+~~~~~~~~~~~~~~~~~
+
+If you want to analyze other data, (for exemple, having the number of HTML
+files), create a custom modules and overload the module model :
+
+.. code-block:: python
+
+  from odoo import api, fields, models
+
+  class IrModuleModule(models.Model):
+     _inherit = 'ir.module.module'
+
+     xml_documentation_qty = fields.Integer(
+        string='Quantity of Comments in XML Files')
+
+    @api.model
+    def _get_analyse_settings(self):
+        res = super(IrModuleModule, self)._get_analyse_settings()
+        if not '.html' in res:
+            res['.html'] = {}
+        res['.html']['documentation'] 'xml_documentation_qty'
+        return res

module_analysis/readme/USAGE.rst

@@ -6,3 +6,4 @@ Open the stats to analyze the detail of the code installed
 
 
     .. image:: ../static/description/analysis_pie.png
+

module_analysis/tests/test_module.py

@@ -17,15 +17,17 @@ class TestModule(TransactionCase):
             [('state', '=', 'installed')])
         for module in installed_modules:
             self.assertTrue(
-                module.python_lines_qty > 0,
-                "module '%s' doesn't have python lines defined, whereas it is"
-                " installed.")
+                module.python_code_qty > 0 or
+                module.xml_code_qty > 0 or
+                module.js_code_qty > 0,
+                "module '%s' doesn't have code analysed defined, whereas it is"
+                " installed." % (module.name))
 
     def test_uninstalled_modules(self):
         uninstalled_modules = self.IrModuleModule.search(
             [('state', '!=', 'installed')])
         for module in uninstalled_modules:
             self.assertTrue(
-                module.python_lines_qty == 0,
+                module.python_code_qty == 0,
                 "module '%s' has python lines defined, whereas it is"
-                " not installed.")
+                " not installed." % (module.name))

module_analysis/views/view_ir_module_module.xml

@@ -14,10 +14,10 @@ License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
             <xpath expr="//field[@name='state']/.." position="after">
                 <button name="button_analyze_code" states="installed" string="Refresh Code Analysis" type="object" class="btn btn-primary"/>
                 <group string="Code Size" col="4">
-                    <field name="python_lines_qty"/>
-                    <field name="xml_lines_qty"/>
-                    <field name="js_lines_qty"/>
-                    <field name="css_lines_qty"/>
+                    <field name="python_code_qty"/>
+                    <field name="xml_code_qty"/>
+                    <field name="js_code_qty"/>
+                    <field name="css_code_qty"/>
                 </group>
                 <group string="Type and Authors">
                     <field name="author_ids" widget="many2many_tags"/>
@@ -33,10 +33,10 @@ License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
         <field name="arch" type="xml">
             <pivot>
                 <field name="module_type_id" type="row"/>
-                <field name="python_lines_qty" type="measure"/>
-                <field name="xml_lines_qty" type="measure"/>
-                <field name="js_lines_qty" type="measure"/>
-                <field name="css_lines_qty" type="measure"/>
+                <field name="python_code_qty" type="measure"/>
+                <field name="xml_code_qty" type="measure"/>
+                <field name="js_code_qty" type="measure"/>
+                <field name="css_code_qty" type="measure"/>
             </pivot>
         </field>
     </record>

requirements.txt

@@ -3,3 +3,4 @@ openpyxl
 xlrd
 xlwt
 pysftp
+pygount