Merge pull request #777 from akretion/8.0-logging-json

[8.0] add module logging json

logging_json/README.rst

@@ -0,0 +1,90 @@
+.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+   :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+   :alt: License: AGPL-3
+
+==============
+JSON Logging
+==============
+
+This addon allows to output the Odoo logs in JSON.
+
+
+It also add the possibility to log extra data in json in your log.
+See documentation of json logger here : https://github.com/madzak/python-json-logger
+
+Example:
+
+.. code-block:: python
+
+   _logger.info('My Message', extra={
+       'my_extra_key_1': 'value_1',
+       'my_extra_key_2': 'value_2'})
+
+The key "extra" is a standard key of logger module so you can add this key and your log will work with or without this module (see documentation here : https://docs.python.org/2/library/logging.html#logging.Logger.debug)
+
+
+Configuration
+=============
+
+The json logging is activated with the environment variable
+``ODOO_LOGGING_JSON`` set to ``1``.
+
+In order to have the logs from the start of the server, you should add
+``logging_json`` in the ``--load`` flag or in the ``server_wide_modules``
+option in the configuration file.
+
+If you want to see the extra keys when you are developping in local
+but with the odoo logger output and not the JSON output
+you can add the environment variable ``ODOO_LOGGING_JSON_DEV`` set to ``1``.
+
+Note : ``ODOO_LOGGING_JSON_DEV`` and ``ODOO_LOGGING_JSON`` should be not set both to ``1``
+
+Known issues / Roadmap
+======================
+
+* Completed the extraction (in regex) of interesting message to get more metric
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues
+<https://github.com/OCA/server-tools/issues>`_. In case of trouble, please
+check there if your issue has already been reported. If you spotted it first,
+help us smash it by providing detailed and welcomed feedback.
+
+Credits
+=======
+
+Images
+------
+
+* Odoo Community Association: `Icon <https://github.com/OCA/maintainer-tools/blob/master/template/module/static/description/icon.svg>`_.
+
+Contributors
+------------
+
+* Guewen Baconnier <guewen.baconnier@camptocamp.com>
+* Sébastien BEAU <sebastien.beau@akretion.com>
+
+Funders
+-------
+
+The development of this module has been financially supported by:
+
+* Camptocamp
+* Akretion
+
+Maintainer
+----------
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+This module is maintained by the OCA.
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+To contribute to this module, please visit https://odoo-community.org.

logging_json/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import json_log

logging_json/__openerp__.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+{'name': 'JSON Logging',
+ 'version': '8.0.1.0.0',
+ 'author': 'Camptocamp,Akretion,Odoo Community Association (OCA)',
+ 'license': 'AGPL-3',
+ 'category': 'Extra Tools',
+ 'depends': ['base',
+             ],
+ 'external_dependencies': {
+     'python': ['pythonjsonlogger'],
+ },
+ 'website': 'http://www.camptocamp.com',
+ 'data': [],
+ 'installable': True,
+ }

logging_json/json_log.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Camptocamp (http://www.camptocamp.com).
+# @author Guewen Baconnier <guewen.baconnier@camptocamp.com>
+# Copyright 2017 Akretion (http://www.akretion.com).
+# @author Sébastien BEAU <sebastien.beau@akretion.com>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+import logging
+import os
+import threading
+import re
+import json
+
+from distutils.util import strtobool
+from openerp.netsvc import ColoredFormatter
+
+_logger = logging.getLogger(__name__)
+
+try:
+    from pythonjsonlogger import jsonlogger
+except ImportError:
+    jsonlogger = None  # noqa
+    _logger.debug("Cannot 'import pythonjsonlogger'.")
+
+
+def is_true(strval):
+    return bool(strtobool(strval or '0'.lower()))
+
+
+# The following regex are use to extract information from native odoo log
+# We struct is the following
+# {name_of_the_log : [regex, formatter, extra_vals]}
+# The extra_vals "dict" will be merged into the jsonlogger if the regex match
+# In order to make understandable the regex please add an string example that
+# match
+REGEX = {
+    'openerp.addons.base.ir.ir_cron': [
+        # "cron.object.execute('db', 1, '*', u'base.action.rule', u'_check')"
+        (r"cron\.object\.execute\('.+'(?P<cron_model>[\w.]+).+"
+         r"'(?P<cron_method>[\w.]+)'\)",
+         {}, {'cron_running': True}),
+
+        # "0.016s (base.action.rule, _check)"
+        (r"(?P<cron_duration_second>[\d.]+)s \("
+         r"(?P<cron_model>[\w.]+), (?P<cron_method>[\w.]+)",
+         {'cron_duration_second': float},
+         {'cron_running': False, 'cron_status': 'succeeded'}),
+
+        # Call of self.pool.get('base.action.rule')._check(
+        # cr, uid, *u'()') failed in Job 43
+        (r"Call of self\.pool\.get\('(?P<cron_model>[\w.]+)'\)"
+         r".(?P<cron_method>[\w.]+)",
+         {}, {'cron_running': False, 'cron_status': 'failed'}),
+    ],
+}
+
+
+class OdooJsonFormatter(jsonlogger.JsonFormatter):
+
+    def add_fields(self, log_record, record, message_dict):
+        record.pid = os.getpid()
+        record.dbname = getattr(threading.currentThread(), 'dbname', '?')
+        _super = super(OdooJsonFormatter, self)
+        res = _super.add_fields(log_record, record, message_dict)
+        if log_record['name'] in REGEX:
+            for regex, formatters, extra_vals in REGEX[log_record['name']]:
+                match = re.match(regex, log_record['message'])
+                if match:
+                    vals = match.groupdict()
+                    for key, func in formatters.items():
+                        vals[key] = func(vals[key])
+                    log_record.update(vals)
+                    if extra_vals:
+                        log_record.update(extra_vals)
+        return res
+
+
+class OdooJsonDevFormatter(ColoredFormatter):
+
+    def format(self, record):
+        response = super(OdooJsonDevFormatter, self).format(record)
+        extra = {}
+        RESERVED_ATTRS = list(jsonlogger.RESERVED_ATTRS) + ["dbname", "pid"]
+        for key, value in record.__dict__.items():
+            if (key not in RESERVED_ATTRS and not
+                    (hasattr(key, "startswith") and key.startswith('_'))):
+                extra[key] = value
+        if extra:
+            response += " extra: " + json.dumps(
+                extra, indent=4, sort_keys=True)
+        return response
+
+
+if is_true(os.environ.get('ODOO_LOGGING_JSON')):
+    format = ('%(asctime)s %(pid)s %(levelname)s'
+              '%(dbname)s %(name)s: %(message)s')
+    formatter = OdooJsonFormatter(format)
+    logging.getLogger().handlers[0].formatter = formatter
+elif is_true(os.environ.get('ODOO_LOGGING_JSON_DEV')):
+    format = ('%(asctime)s %(pid)s %(levelname)s'
+              '%(dbname)s %(name)s: %(message)s')
+    formatter = OdooJsonDevFormatter(format)
+    logging.getLogger().handlers[0].formatter = formatter

requirements.txt

@@ -5,3 +5,4 @@
   pysftp
   acme_tiny
   IPy
+  python-json-logger