OCA-git-bot
4 years ago
19 changed files with 1334 additions and 317 deletions
-
2requirements.txt
-
168sentry/README.rst
-
83sentry/__init__.py
-
7sentry/__manifest__.py
-
103sentry/const.py
-
63sentry/generalutils.py
-
113sentry/hooks.py
-
129sentry/logutils.py
-
137sentry/processor.py
-
3sentry/readme/CONFIGURE.rst
-
4sentry/readme/CONTRIBUTORS.rst
-
1sentry/readme/CREDITS.rst
-
76sentry/readme/DESCRIPTION.rst
-
10sentry/readme/INSTALL.rst
-
11sentry/readme/ROADMAP.rst
-
6sentry/readme/USAGE.rst
-
579sentry/static/description/index.html
-
142sentry/tests/test_client.py
-
14sentry/tests/test_logutils.py
@ -1,82 +1 @@ |
|||
# Copyright 2016-2017 Versada <https://versada.eu/> |
|||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). |
|||
|
|||
import logging |
|||
|
|||
from odoo.service import wsgi_server |
|||
from odoo.tools import config as odoo_config |
|||
|
|||
from . import const |
|||
from .logutils import LoggerNameFilter, OdooSentryHandler |
|||
|
|||
import collections |
|||
|
|||
_logger = logging.getLogger(__name__) |
|||
HAS_RAVEN = True |
|||
try: |
|||
import raven |
|||
from raven.middleware import Sentry |
|||
except ImportError: |
|||
HAS_RAVEN = False |
|||
_logger.debug('Cannot import "raven". Please make sure it is installed.') |
|||
|
|||
|
|||
def get_odoo_commit(odoo_dir): |
|||
'''Attempts to get Odoo git commit from :param:`odoo_dir`.''' |
|||
if not odoo_dir: |
|||
return |
|||
try: |
|||
return raven.fetch_git_sha(odoo_dir) |
|||
except raven.exceptions.InvalidGitRepository: |
|||
_logger.debug( |
|||
'Odoo directory: "%s" not a valid git repository', odoo_dir) |
|||
|
|||
|
|||
def initialize_raven(config, client_cls=None): |
|||
''' |
|||
Setup an instance of :class:`raven.Client`. |
|||
|
|||
:param config: Sentry configuration |
|||
:param client: class used to instantiate the raven client. |
|||
''' |
|||
enabled = config.get('sentry_enabled', False) |
|||
if not (HAS_RAVEN and enabled): |
|||
return |
|||
|
|||
if config.get('sentry_odoo_dir') and config.get('sentry_release'): |
|||
_logger.debug('Both sentry_odoo_dir and sentry_release defined, choosing sentry_release') |
|||
options = { |
|||
'release': config.get('sentry_release', get_odoo_commit(config.get('sentry_odoo_dir'))), |
|||
} |
|||
for option in const.get_sentry_options(): |
|||
value = config.get('sentry_%s' % option.key, option.default) |
|||
if isinstance(option.converter, collections.Callable): |
|||
value = option.converter(value) |
|||
options[option.key] = value |
|||
|
|||
level = config.get('sentry_logging_level', const.DEFAULT_LOG_LEVEL) |
|||
exclude_loggers = const.split_multiple( |
|||
config.get('sentry_exclude_loggers', const.DEFAULT_EXCLUDE_LOGGERS) |
|||
) |
|||
if level not in const.LOG_LEVEL_MAP: |
|||
level = const.DEFAULT_LOG_LEVEL |
|||
|
|||
client_cls = client_cls or raven.Client |
|||
client = client_cls(**options) |
|||
handler = OdooSentryHandler( |
|||
config.get('sentry_include_context', True), |
|||
client=client, |
|||
level=const.LOG_LEVEL_MAP[level], |
|||
) |
|||
if exclude_loggers: |
|||
handler.addFilter(LoggerNameFilter( |
|||
exclude_loggers, name='sentry.logger.filter')) |
|||
raven.conf.setup_logging(handler) |
|||
wsgi_server.application = Sentry( |
|||
wsgi_server.application, client=client) |
|||
|
|||
client.captureMessage('Starting Odoo Server') |
|||
return client |
|||
|
|||
|
|||
sentry_client = initialize_raven(odoo_config) |
|||
from .hooks import post_load |
|||
@ -0,0 +1,63 @@ |
|||
try: |
|||
from collections.abc import Mapping |
|||
except ImportError: # pragma: no cover |
|||
# Python < 3.3 |
|||
from collections import Mapping # pragma: no cover |
|||
|
|||
|
|||
def string_types(): |
|||
""" Taken from https://git.io/JIv5J """ |
|||
|
|||
return str, |
|||
|
|||
|
|||
def is_namedtuple(value): |
|||
""" https://stackoverflow.com/a/2166841/1843746 |
|||
But modified to handle subclasses of namedtuples. |
|||
Taken from https://git.io/JIsfY |
|||
""" |
|||
if not isinstance(value, tuple): |
|||
return False |
|||
f = getattr(type(value), '_fields', None) |
|||
if not isinstance(f, tuple): |
|||
return False |
|||
return all(type(n) == str for n in f) |
|||
|
|||
|
|||
def iteritems(d, **kw): |
|||
""" Override iteritems for support multiple versions python. |
|||
Taken from https://git.io/JIvMi |
|||
""" |
|||
return iter(d.items(**kw)) |
|||
|
|||
|
|||
def varmap(func, var, context=None, name=None): |
|||
""" Executes ``func(key_name, value)`` on all values |
|||
recurisively discovering dict and list scoped |
|||
values. Taken from https://git.io/JIvMN |
|||
""" |
|||
if context is None: |
|||
context = {} |
|||
objid = id(var) |
|||
if objid in context: |
|||
return func(name, '<...>') |
|||
context[objid] = 1 |
|||
|
|||
if isinstance(var, (list, tuple)) and not is_namedtuple(var): |
|||
ret = [varmap(func, f, context, name) for f in var] |
|||
else: |
|||
ret = func(name, var) |
|||
if isinstance(ret, Mapping): |
|||
ret = dict((k, varmap(func, v, context, k)) |
|||
for k, v in iteritems(var)) |
|||
del context[objid] |
|||
return ret |
|||
|
|||
|
|||
def get_environ(environ): |
|||
""" Returns our whitelisted environment variables. |
|||
Taken from https://git.io/JIsf2 |
|||
""" |
|||
for key in ('REMOTE_ADDR', 'SERVER_NAME', 'SERVER_PORT'): |
|||
if key in environ: |
|||
yield key, environ[key] |
|||
@ -0,0 +1,113 @@ |
|||
# Copyright 2016-2017 Versada <https://versada.eu/> |
|||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). |
|||
|
|||
import logging |
|||
|
|||
import odoo.http |
|||
from odoo.service import wsgi_server |
|||
from odoo.tools import config as odoo_config |
|||
|
|||
from . import const |
|||
from .logutils import SanitizeOdooCookiesProcessor, fetch_git_sha |
|||
from .logutils import InvalidGitRepository, get_extra_context |
|||
|
|||
import collections |
|||
|
|||
_logger = logging.getLogger(__name__) |
|||
HAS_SENTRY_SDK = True |
|||
try: |
|||
import sentry_sdk |
|||
from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware |
|||
from sentry_sdk.integrations.threading import ThreadingIntegration |
|||
from sentry_sdk.integrations.logging import ignore_logger |
|||
except ImportError: # pragma: no cover |
|||
HAS_SENTRY_SDK = False # pragma: no cover |
|||
_logger.debug("Cannot import 'sentry-sdk'.\ |
|||
Please make sure it is installed.") # pragma: no cover |
|||
|
|||
|
|||
def before_send(event, hint): |
|||
""" Add context to event if include_context is True |
|||
and sanitize sensitive data """ |
|||
if event.setdefault("tags", {})["include_context"]: |
|||
cxtest = get_extra_context(odoo.http.request) |
|||
info_request = ["tags", "user", "extra", "request"] |
|||
|
|||
for item in info_request: |
|||
info_item = event.setdefault(item, {}) |
|||
info_item.update(cxtest.setdefault(item, {})) |
|||
|
|||
raven_processor = SanitizeOdooCookiesProcessor() |
|||
raven_processor.process(event) |
|||
|
|||
return event |
|||
|
|||
|
|||
def get_odoo_commit(odoo_dir): |
|||
"""Attempts to get Odoo git commit from :param:`odoo_dir`.""" |
|||
if not odoo_dir: |
|||
return |
|||
try: |
|||
return fetch_git_sha(odoo_dir) |
|||
except InvalidGitRepository: |
|||
_logger.debug( |
|||
"Odoo directory: '%s' not a valid git repository", odoo_dir) |
|||
|
|||
|
|||
def initialize_sentry(config): |
|||
""" Setup an instance of :class:`sentry_sdk.Client`. |
|||
:param config: Sentry configuration |
|||
:param client: class used to instantiate the sentry_sdk client. |
|||
""" |
|||
enabled = config.get("sentry_enabled", False) |
|||
if not (HAS_SENTRY_SDK and enabled): |
|||
return |
|||
_logger.info("Initializing sentry...") |
|||
if config.get("sentry_odoo_dir") and config.get("sentry_release"): |
|||
_logger.debug("Both sentry_odoo_dir and \ |
|||
sentry_release defined, choosing sentry_release") |
|||
options = { |
|||
"release": config.get("sentry_release", |
|||
get_odoo_commit(config.get("sentry_odoo_dir"))), |
|||
} |
|||
for option in const.get_sentry_options(): |
|||
value = config.get('sentry_%s' % option.key, option.default) |
|||
if isinstance(option.converter, collections.Callable): |
|||
value = option.converter(value) |
|||
options[option.key] = value |
|||
|
|||
exclude_loggers = const.split_multiple( |
|||
config.get("sentry_exclude_loggers", const.DEFAULT_EXCLUDE_LOGGERS) |
|||
) |
|||
# Change name `ignore_exceptions` (with raven) |
|||
# to `ignore_errors' (sentry_sdk) |
|||
options["ignore_errors"] = options["ignore_exceptions"] |
|||
del options["ignore_exceptions"] |
|||
|
|||
options["before_send"] = before_send |
|||
|
|||
options["integrations"] = [options["logging_level"], |
|||
ThreadingIntegration(propagate_hub=True)] |
|||
# Remove logging_level, since in sentry_sdk is include in 'integrations' |
|||
del options["logging_level"] |
|||
|
|||
client = sentry_sdk.init(**options) |
|||
|
|||
sentry_sdk.set_tag("include_context", |
|||
config.get("sentry_include_context", True)) |
|||
|
|||
if exclude_loggers: |
|||
for item in exclude_loggers: |
|||
ignore_logger(item) |
|||
|
|||
wsgi_server.application = SentryWsgiMiddleware(wsgi_server.application) |
|||
|
|||
with sentry_sdk.push_scope() as scope: |
|||
scope.set_extra("debug", False) |
|||
sentry_sdk.capture_message("Starting Odoo Server", "info") |
|||
|
|||
return client |
|||
|
|||
|
|||
def post_load(): |
|||
initialize_sentry(odoo_config) |
|||
@ -0,0 +1,137 @@ |
|||
""" Custom class of raven.core.processors taken of https://git.io/JITko |
|||
This is a custom class of processor to filter and sanitize |
|||
passwords and keys from request data, it does not exist in |
|||
sentry-sdk. |
|||
""" |
|||
|
|||
from __future__ import absolute_import |
|||
|
|||
import re |
|||
|
|||
from sentry_sdk._compat import text_type |
|||
from .generalutils import string_types, varmap |
|||
|
|||
|
|||
class SanitizeKeysProcessor(object): |
|||
""" Class from raven for sanitize keys, cookies, etc |
|||
Asterisk out things that correspond to a configurable set of keys. """ |
|||
|
|||
MASK = '*' * 8 |
|||
|
|||
def process(self, data, **kwargs): |
|||
if 'exception' in data: |
|||
if 'values' in data['exception']: |
|||
for value in data['exception'].get('values', []): |
|||
if 'stacktrace' in value: |
|||
self.filter_stacktrace(value['stacktrace']) |
|||
|
|||
if 'request' in data: |
|||
self.filter_http(data['request']) |
|||
|
|||
if 'extra' in data: |
|||
data['extra'] = self.filter_extra(data['extra']) |
|||
|
|||
if 'level' in data: |
|||
data['level'] = self.filter_level(data['level']) |
|||
|
|||
return data |
|||
|
|||
@property |
|||
def sanitize_keys(self): |
|||
pass |
|||
|
|||
def sanitize(self, item, value): |
|||
if value is None: |
|||
return |
|||
|
|||
if not item: # key can be a NoneType |
|||
return value |
|||
|
|||
# Just in case we have bytes here, we want to make them into text |
|||
# properly without failing so we can perform our check. |
|||
if isinstance(item, bytes): |
|||
item = item.decode('utf-8', 'replace') |
|||
else: |
|||
item = text_type(item) |
|||
|
|||
item = item.lower() |
|||
for key in self.sanitize_keys: |
|||
if key in item: |
|||
# store mask as a fixed length for security |
|||
return self.MASK |
|||
return value |
|||
|
|||
def filter_stacktrace(self, data): |
|||
for frame in data.get('frames', []): |
|||
if 'vars' not in frame: |
|||
continue |
|||
frame['vars'] = varmap(self.sanitize, frame['vars']) |
|||
|
|||
def filter_http(self, data): |
|||
for n in ('data', 'cookies', 'headers', 'env', 'query_string'): |
|||
if n not in data: |
|||
continue |
|||
|
|||
# data could be provided as bytes and if it's python3 |
|||
if isinstance(data[n], bytes): |
|||
data[n] = data[n].decode('utf-8', 'replace') |
|||
|
|||
if isinstance(data[n], string_types()) and '=' in data[n]: |
|||
# at this point we've assumed it's a standard HTTP query |
|||
# or cookie |
|||
if n == 'cookies': |
|||
delimiter = ';' |
|||
else: |
|||
delimiter = '&' |
|||
|
|||
data[n] = self._sanitize_keyvals(data[n], delimiter) |
|||
else: |
|||
data[n] = varmap(self.sanitize, data[n]) |
|||
if n == 'headers' and 'Cookie' in data[n]: |
|||
data[n]['Cookie'] = self._sanitize_keyvals( |
|||
data[n]['Cookie'], ';' |
|||
) |
|||
|
|||
def filter_extra(self, data): |
|||
return varmap(self.sanitize, data) |
|||
|
|||
def filter_level(self, data): |
|||
return re.sub(r'\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))', '', data) |
|||
|
|||
def _sanitize_keyvals(self, keyvals, delimiter): |
|||
sanitized_keyvals = [] |
|||
for keyval in keyvals.split(delimiter): |
|||
keyval = keyval.split('=') |
|||
if len(keyval) == 2: |
|||
sanitized_keyvals.append((keyval[0], self.sanitize(*keyval))) |
|||
else: |
|||
sanitized_keyvals.append(keyval) |
|||
|
|||
return delimiter.join('='.join(keyval) for keyval in sanitized_keyvals) |
|||
|
|||
|
|||
class SanitizePasswordsProcessor(SanitizeKeysProcessor): |
|||
""" Asterisk out things that look like passwords, credit card numbers, |
|||
and API keys in frames, http, and basic extra data. """ |
|||
|
|||
KEYS = frozenset([ |
|||
'password', |
|||
'secret', |
|||
'passwd', |
|||
'authorization', |
|||
'api_key', |
|||
'apikey', |
|||
'sentry_dsn', |
|||
'access_token', |
|||
]) |
|||
VALUES_RE = re.compile(r'^(?:\d[ -]*?){13,16}$') |
|||
|
|||
@property |
|||
def sanitize_keys(self): |
|||
return self.KEYS |
|||
|
|||
def sanitize(self, item, value): |
|||
value = super(SanitizePasswordsProcessor, self).sanitize(item, value) |
|||
if isinstance(value, string_types()) and self.VALUES_RE.match(value): |
|||
return self.MASK |
|||
return value |
|||
@ -0,0 +1,3 @@ |
|||
The following additional configuration options can be added to your Odoo |
|||
configuration file: |
|||
|
|||
@ -0,0 +1,4 @@ |
|||
* Mohammed Barsi <barsintod@gmail.com> |
|||
* Andrius Preimantas <andrius@versada.eu> |
|||
* Naglis Jonaitis <naglis@versada.eu> |
|||
* Atte Isopuro <atte.isopuro@avoin.systems> |
|||
@ -0,0 +1 @@ |
|||
* Vauxoo |
|||
@ -0,0 +1,76 @@ |
|||
This module allows painless `Sentry <https://sentry.io/>`__ integration with |
|||
Odoo. |
|||
|
|||
============================= ==================================================================== ========================================================== |
|||
Option Description Default |
|||
============================= ==================================================================== ========================================================== |
|||
``sentry_dsn`` Sentry *Data Source Name*. You can find this value in your Sentry ``''`` |
|||
project configuration. Typically it looks something like this: |
|||
*https://<public_key>:<secret_key>@sentry.example.com/<project id>* |
|||
This is the only required option in order to use the module. |
|||
|
|||
``sentry_enabled`` Whether or not Sentry logging is enabled. ``False`` |
|||
|
|||
``sentry_logging_level`` The minimal logging level for which to send reports to Sentry. ``warn`` |
|||
Possible values: *notset*, *debug*, *info*, *warn*, *error*, |
|||
*critical*. It is recommended to have this set to at least *warn*, |
|||
to avoid spamming yourself with Sentry events. |
|||
|
|||
``sentry_exclude_loggers`` A string of comma-separated logger names which should be excluded ``werkzeug`` |
|||
from Sentry. |
|||
|
|||
``sentry_ignored_exceptions`` A string of comma-separated exceptions which should be ignored. ``odoo.exceptions.AccessDenied, |
|||
You can use a star symbol (*) at the end, to ignore all exceptions odoo.exceptions.AccessError, |
|||
from a module, eg.: *odoo.exceptions.**. odoo.exceptions.DeferredException, |
|||
odoo.exceptions.MissingError, |
|||
odoo.exceptions.RedirectWarning, |
|||
odoo.exceptions.UserError, |
|||
odoo.exceptions.ValidationError, |
|||
odoo.exceptions.Warning, |
|||
odoo.exceptions.except_orm`` |
|||
|
|||
``sentry_include_context`` If enabled, additional context data will be extracted from current ``True`` |
|||
HTTP request and user session (if available). This has no effect |
|||
for Cron jobs, as no request/session is available inside a Cron job. |
|||
|
|||
``sentry_release`` Explicitly define a version to be sent as the release version to |
|||
Sentry. Useful in conjuntion with Sentry's "Resolve in the next |
|||
release"-functionality. Also useful if your production deployment |
|||
does not include any Git context from which a commit might be read. |
|||
Overrides *sentry_odoo_dir*. |
|||
|
|||
``sentry_odoo_dir`` Absolute path to your Odoo installation directory. This is optional |
|||
and will only be used to extract the Odoo Git commit, which will be |
|||
sent to Sentry, to allow to distinguish between Odoo updates. |
|||
Overridden by *sentry_release* |
|||
============================= ==================================================================== ========================================================== |
|||
|
|||
Other `client arguments |
|||
<https://docs.sentry.io/platforms/python/configuration/>`_ can be |
|||
configured by prepending the argument name with *sentry_* in your Odoo config |
|||
file. Currently supported additional client arguments are: ``with_locals, |
|||
max_breadcrumbs, release, environment, server_name, shutdown_timeout, |
|||
in_app_include, in_app_exclude, default_integrations, dist, sample_rate, |
|||
send_default_pii, http_proxy, https_proxy, request_bodies, debug, |
|||
attach_stacktrace, ca_certs, propagate_traces, traces_sample_rate, |
|||
auto_enabling_integrations``. |
|||
|
|||
Example Odoo configuration |
|||
-------------------------- |
|||
|
|||
Below is an example of Odoo configuration file with *Odoo Sentry* options:: |
|||
|
|||
[options] |
|||
sentry_dsn = https://<public_key>:<secret_key>@sentry.example.com/<project id> |
|||
sentry_enabled = true |
|||
sentry_logging_level = warn |
|||
sentry_exclude_loggers = werkzeug |
|||
sentry_ignore_exceptions = odoo.exceptions.AccessDenied, |
|||
odoo.exceptions.AccessError,odoo.exceptions.MissingError, |
|||
odoo.exceptions.RedirectWarning,odoo.exceptions.UserError, |
|||
odoo.exceptions.ValidationError,odoo.exceptions.Warning, |
|||
odoo.exceptions.except_orm |
|||
sentry_include_context = true |
|||
sentry_environment = production |
|||
sentry_release = 1.3.2 |
|||
sentry_odoo_dir = /home/odoo/odoo/ |
|||
@ -0,0 +1,10 @@ |
|||
The module can be installed just like any other Odoo module, by adding the |
|||
module's directory to Odoo *addons_path*. In order for the module to correctly |
|||
wrap the Odoo WSGI application, it also needs to be loaded as a server-wide |
|||
module. This can be done with the ``server_wide_modules`` parameter in your |
|||
Odoo config file or with the ``--load`` command-line parameter. |
|||
|
|||
This module additionally requires the sentry-sdk Python package to be available on |
|||
the system. It can be installed using pip:: |
|||
|
|||
pip install sentry-sdk |
|||
@ -0,0 +1,11 @@ |
|||
* **No database separation** -- This module functions by intercepting all Odoo |
|||
logging records in a running Odoo process. This means that once installed in |
|||
one database, it will intercept and report errors for all Odoo databases, |
|||
which are used on that Odoo server. |
|||
|
|||
* **Frontend integration** -- In the future, it would be nice to add |
|||
Odoo client-side error reporting to this module as well, by integrating |
|||
`raven-js <https://github.com/getsentry/raven-js>`_. Additionally, `Sentry user |
|||
feedback form <https://docs.sentry.io/learn/user-feedback/>`_ could be |
|||
integrated into the Odoo client error dialog window to allow users shortly |
|||
describe what they were doing when things went wrong. |
|||
@ -0,0 +1,6 @@ |
|||
Once configured and installed, the module will report any logging event at and |
|||
above the configured Sentry logging level, no additional actions are necessary. |
|||
|
|||
.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas |
|||
:alt: Try me on Runbot |
|||
:target: https://runbot.odoo-community.org/runbot/149/10.0 |
|||
@ -0,0 +1,579 @@ |
|||
<?xml version="1.0" encoding="utf-8" ?> |
|||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
|||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> |
|||
<head> |
|||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
|||
<meta name="generator" content="Docutils 0.15.1: http://docutils.sourceforge.net/" /> |
|||
<title>Sentry</title> |
|||
<style type="text/css"> |
|||
|
|||
/* |
|||
:Author: David Goodger (goodger@python.org) |
|||
:Id: $Id: html4css1.css 7952 2016-07-26 18:15:59Z milde $ |
|||
:Copyright: This stylesheet has been placed in the public domain. |
|||
|
|||
Default cascading style sheet for the HTML output of Docutils. |
|||
|
|||
See http://docutils.sf.net/docs/howto/html-stylesheets.html for how to |
|||
customize this style sheet. |
|||
*/ |
|||
|
|||
/* used to remove borders from tables and images */ |
|||
.borderless, table.borderless td, table.borderless th { |
|||
border: 0 } |
|||
|
|||
table.borderless td, table.borderless th { |
|||
/* Override padding for "table.docutils td" with "! important". |
|||
The right padding separates the table cells. */ |
|||
padding: 0 0.5em 0 0 ! important } |
|||
|
|||
.first { |
|||
/* Override more specific margin styles with "! important". */ |
|||
margin-top: 0 ! important } |
|||
|
|||
.last, .with-subtitle { |
|||
margin-bottom: 0 ! important } |
|||
|
|||
.hidden { |
|||
display: none } |
|||
|
|||
.subscript { |
|||
vertical-align: sub; |
|||
font-size: smaller } |
|||
|
|||
.superscript { |
|||
vertical-align: super; |
|||
font-size: smaller } |
|||
|
|||
a.toc-backref { |
|||
text-decoration: none ; |
|||
color: black } |
|||
|
|||
blockquote.epigraph { |
|||
margin: 2em 5em ; } |
|||
|
|||
dl.docutils dd { |
|||
margin-bottom: 0.5em } |
|||
|
|||
object[type="image/svg+xml"], object[type="application/x-shockwave-flash"] { |
|||
overflow: hidden; |
|||
} |
|||
|
|||
/* Uncomment (and remove this text!) to get bold-faced definition list terms |
|||
dl.docutils dt { |
|||
font-weight: bold } |
|||
*/ |
|||
|
|||
div.abstract { |
|||
margin: 2em 5em } |
|||
|
|||
div.abstract p.topic-title { |
|||
font-weight: bold ; |
|||
text-align: center } |
|||
|
|||
div.admonition, div.attention, div.caution, div.danger, div.error, |
|||
div.hint, div.important, div.note, div.tip, div.warning { |
|||
margin: 2em ; |
|||
border: medium outset ; |
|||
padding: 1em } |
|||
|
|||
div.admonition p.admonition-title, div.hint p.admonition-title, |
|||
div.important p.admonition-title, div.note p.admonition-title, |
|||
div.tip p.admonition-title { |
|||
font-weight: bold ; |
|||
font-family: sans-serif } |
|||
|
|||
div.attention p.admonition-title, div.caution p.admonition-title, |
|||
div.danger p.admonition-title, div.error p.admonition-title, |
|||
div.warning p.admonition-title, .code .error { |
|||
color: red ; |
|||
font-weight: bold ; |
|||
font-family: sans-serif } |
|||
|
|||
/* Uncomment (and remove this text!) to get reduced vertical space in |
|||
compound paragraphs. |
|||
div.compound .compound-first, div.compound .compound-middle { |
|||
margin-bottom: 0.5em } |
|||
|
|||
div.compound .compound-last, div.compound .compound-middle { |
|||
margin-top: 0.5em } |
|||
*/ |
|||
|
|||
div.dedication { |
|||
margin: 2em 5em ; |
|||
text-align: center ; |
|||
font-style: italic } |
|||
|
|||
div.dedication p.topic-title { |
|||
font-weight: bold ; |
|||
font-style: normal } |
|||
|
|||
div.figure { |
|||
margin-left: 2em ; |
|||
margin-right: 2em } |
|||
|
|||
div.footer, div.header { |
|||
clear: both; |
|||
font-size: smaller } |
|||
|
|||
div.line-block { |
|||
display: block ; |
|||
margin-top: 1em ; |
|||
margin-bottom: 1em } |
|||
|
|||
div.line-block div.line-block { |
|||
margin-top: 0 ; |
|||
margin-bottom: 0 ; |
|||
margin-left: 1.5em } |
|||
|
|||
div.sidebar { |
|||
margin: 0 0 0.5em 1em ; |
|||
border: medium outset ; |
|||
padding: 1em ; |
|||
background-color: #ffffee ; |
|||
width: 40% ; |
|||
float: right ; |
|||
clear: right } |
|||
|
|||
div.sidebar p.rubric { |
|||
font-family: sans-serif ; |
|||
font-size: medium } |
|||
|
|||
div.system-messages { |
|||
margin: 5em } |
|||
|
|||
div.system-messages h1 { |
|||
color: red } |
|||
|
|||
div.system-message { |
|||
border: medium outset ; |
|||
padding: 1em } |
|||
|
|||
div.system-message p.system-message-title { |
|||
color: red ; |
|||
font-weight: bold } |
|||
|
|||
div.topic { |
|||
margin: 2em } |
|||
|
|||
h1.section-subtitle, h2.section-subtitle, h3.section-subtitle, |
|||
h4.section-subtitle, h5.section-subtitle, h6.section-subtitle { |
|||
margin-top: 0.4em } |
|||
|
|||
h1.title { |
|||
text-align: center } |
|||
|
|||
h2.subtitle { |
|||
text-align: center } |
|||
|
|||
hr.docutils { |
|||
width: 75% } |
|||
|
|||
img.align-left, .figure.align-left, object.align-left, table.align-left { |
|||
clear: left ; |
|||
float: left ; |
|||
margin-right: 1em } |
|||
|
|||
img.align-right, .figure.align-right, object.align-right, table.align-right { |
|||
clear: right ; |
|||
float: right ; |
|||
margin-left: 1em } |
|||
|
|||
img.align-center, .figure.align-center, object.align-center { |
|||
display: block; |
|||
margin-left: auto; |
|||
margin-right: auto; |
|||
} |
|||
|
|||
table.align-center { |
|||
margin-left: auto; |
|||
margin-right: auto; |
|||
} |
|||
|
|||
.align-left { |
|||
text-align: left } |
|||
|
|||
.align-center { |
|||
clear: both ; |
|||
text-align: center } |
|||
|
|||
.align-right { |
|||
text-align: right } |
|||
|
|||
/* reset inner alignment in figures */ |
|||
div.align-right { |
|||
text-align: inherit } |
|||
|
|||
/* div.align-center * { */ |
|||
/* text-align: left } */ |
|||
|
|||
.align-top { |
|||
vertical-align: top } |
|||
|
|||
.align-middle { |
|||
vertical-align: middle } |
|||
|
|||
.align-bottom { |
|||
vertical-align: bottom } |
|||
|
|||
ol.simple, ul.simple { |
|||
margin-bottom: 1em } |
|||
|
|||
ol.arabic { |
|||
list-style: decimal } |
|||
|
|||
ol.loweralpha { |
|||
list-style: lower-alpha } |
|||
|
|||
ol.upperalpha { |
|||
list-style: upper-alpha } |
|||
|
|||
ol.lowerroman { |
|||
list-style: lower-roman } |
|||
|
|||
ol.upperroman { |
|||
list-style: upper-roman } |
|||
|
|||
p.attribution { |
|||
text-align: right ; |
|||
margin-left: 50% } |
|||
|
|||
p.caption { |
|||
font-style: italic } |
|||
|
|||
p.credits { |
|||
font-style: italic ; |
|||
font-size: smaller } |
|||
|
|||
p.label { |
|||
white-space: nowrap } |
|||
|
|||
p.rubric { |
|||
font-weight: bold ; |
|||
font-size: larger ; |
|||
color: maroon ; |
|||
text-align: center } |
|||
|
|||
p.sidebar-title { |
|||
font-family: sans-serif ; |
|||
font-weight: bold ; |
|||
font-size: larger } |
|||
|
|||
p.sidebar-subtitle { |
|||
font-family: sans-serif ; |
|||
font-weight: bold } |
|||
|
|||
p.topic-title { |
|||
font-weight: bold } |
|||
|
|||
pre.address { |
|||
margin-bottom: 0 ; |
|||
margin-top: 0 ; |
|||
font: inherit } |
|||
|
|||
pre.literal-block, pre.doctest-block, pre.math, pre.code { |
|||
margin-left: 2em ; |
|||
margin-right: 2em } |
|||
|
|||
pre.code .ln { color: grey; } /* line numbers */ |
|||
pre.code, code { background-color: #eeeeee } |
|||
pre.code .comment, code .comment { color: #5C6576 } |
|||
pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold } |
|||
pre.code .literal.string, code .literal.string { color: #0C5404 } |
|||
pre.code .name.builtin, code .name.builtin { color: #352B84 } |
|||
pre.code .deleted, code .deleted { background-color: #DEB0A1} |
|||
pre.code .inserted, code .inserted { background-color: #A3D289} |
|||
|
|||
span.classifier { |
|||
font-family: sans-serif ; |
|||
font-style: oblique } |
|||
|
|||
span.classifier-delimiter { |
|||
font-family: sans-serif ; |
|||
font-weight: bold } |
|||
|
|||
span.interpreted { |
|||
font-family: sans-serif } |
|||
|
|||
span.option { |
|||
white-space: nowrap } |
|||
|
|||
span.pre { |
|||
white-space: pre } |
|||
|
|||
span.problematic { |
|||
color: red } |
|||
|
|||
span.section-subtitle { |
|||
/* font-size relative to parent (h1..h6 element) */ |
|||
font-size: 80% } |
|||
|
|||
table.citation { |
|||
border-left: solid 1px gray; |
|||
margin-left: 1px } |
|||
|
|||
table.docinfo { |
|||
margin: 2em 4em } |
|||
|
|||
table.docutils { |
|||
margin-top: 0.5em ; |
|||
margin-bottom: 0.5em } |
|||
|
|||
table.footnote { |
|||
border-left: solid 1px black; |
|||
margin-left: 1px } |
|||
|
|||
table.docutils td, table.docutils th, |
|||
table.docinfo td, table.docinfo th { |
|||
padding-left: 0.5em ; |
|||
padding-right: 0.5em ; |
|||
vertical-align: top } |
|||
|
|||
table.docutils th.field-name, table.docinfo th.docinfo-name { |
|||
font-weight: bold ; |
|||
text-align: left ; |
|||
white-space: nowrap ; |
|||
padding-left: 0 } |
|||
|
|||
/* "booktabs" style (no vertical lines) */ |
|||
table.docutils.booktabs { |
|||
border: 0px; |
|||
border-top: 2px solid; |
|||
border-bottom: 2px solid; |
|||
border-collapse: collapse; |
|||
} |
|||
table.docutils.booktabs * { |
|||
border: 0px; |
|||
} |
|||
table.docutils.booktabs th { |
|||
border-bottom: thin solid; |
|||
text-align: left; |
|||
} |
|||
|
|||
h1 tt.docutils, h2 tt.docutils, h3 tt.docutils, |
|||
h4 tt.docutils, h5 tt.docutils, h6 tt.docutils { |
|||
font-size: 100% } |
|||
|
|||
ul.auto-toc { |
|||
list-style-type: none } |
|||
|
|||
</style> |
|||
</head> |
|||
<body> |
|||
<div class="document" id="sentry"> |
|||
<h1 class="title">Sentry</h1> |
|||
|
|||
<!-- !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! |
|||
!! This file is generated by oca-gen-addon-readme !! |
|||
!! changes will be overwritten. !! |
|||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> |
|||
<p><a class="reference external" href="https://odoo-community.org/page/development-status"><img alt="Beta" src="https://img.shields.io/badge/maturity-Beta-yellow.png" /></a> <a class="reference external" href="http://www.gnu.org/licenses/agpl-3.0-standalone.html"><img alt="License: AGPL-3" src="https://img.shields.io/badge/licence-AGPL--3-blue.png" /></a> <a class="reference external" href="https://github.com/OCA/server-tools/tree/12.0-migsentrynewapi-fer/sentry"><img alt="OCA/server-tools" src="https://img.shields.io/badge/github-OCA%2Fserver--tools-lightgray.png?logo=github" /></a> <a class="reference external" href="https://translation.odoo-community.org/projects/server-tools-12-0-migsentrynewapi-fer/server-tools-12-0-migsentrynewapi-fer-sentry"><img alt="Translate me on Weblate" src="https://img.shields.io/badge/weblate-Translate%20me-F47D42.png" /></a> <a class="reference external" href="https://runbot.odoo-community.org/runbot/149/12.0-migsentrynewapi-fer"><img alt="Try me on Runbot" src="https://img.shields.io/badge/runbot-Try%20me-875A7B.png" /></a></p> |
|||
<p>This module allows painless <a class="reference external" href="https://sentry.io/">Sentry</a> integration with |
|||
Odoo.</p> |
|||
<table border="1" class="docutils"> |
|||
<colgroup> |
|||
<col width="19%" /> |
|||
<col width="44%" /> |
|||
<col width="37%" /> |
|||
</colgroup> |
|||
<thead valign="bottom"> |
|||
<tr><th class="head">Option</th> |
|||
<th class="head">Description</th> |
|||
<th class="head">Default</th> |
|||
</tr> |
|||
</thead> |
|||
<tbody valign="top"> |
|||
<tr><td><tt class="docutils literal">sentry_dsn</tt></td> |
|||
<td>Sentry <em>Data Source Name</em>. You can find this value in your Sentry |
|||
project configuration. Typically it looks something like this: |
|||
<em>https://<public_key>:<secret_key>@sentry.example.com/<project id></em> |
|||
This is the only required option in order to use the module.</td> |
|||
<td><tt class="docutils literal">''</tt></td> |
|||
</tr> |
|||
<tr><td><tt class="docutils literal">sentry_enabled</tt></td> |
|||
<td>Whether or not Sentry logging is enabled.</td> |
|||
<td><tt class="docutils literal">False</tt></td> |
|||
</tr> |
|||
<tr><td><tt class="docutils literal">sentry_logging_level</tt></td> |
|||
<td>The minimal logging level for which to send reports to Sentry. |
|||
Possible values: <em>notset</em>, <em>debug</em>, <em>info</em>, <em>warn</em>, <em>error</em>, |
|||
<em>critical</em>. It is recommended to have this set to at least <em>warn</em>, |
|||
to avoid spamming yourself with Sentry events.</td> |
|||
<td><tt class="docutils literal">warn</tt></td> |
|||
</tr> |
|||
<tr><td><tt class="docutils literal">sentry_exclude_loggers</tt></td> |
|||
<td>A string of comma-separated logger names which should be excluded |
|||
from Sentry.</td> |
|||
<td><tt class="docutils literal">werkzeug</tt></td> |
|||
</tr> |
|||
<tr><td><tt class="docutils literal">sentry_ignored_exceptions</tt></td> |
|||
<td>A string of comma-separated exceptions which should be ignored. |
|||
You can use a star symbol (*) at the end, to ignore all exceptions |
|||
from a module, eg.: <em>odoo.exceptions.*</em>.</td> |
|||
<td><tt class="docutils literal">odoo.exceptions.AccessDenied, |
|||
odoo.exceptions.AccessError, |
|||
odoo.exceptions.DeferredException, |
|||
odoo.exceptions.MissingError, |
|||
odoo.exceptions.RedirectWarning, |
|||
odoo.exceptions.UserError, |
|||
odoo.exceptions.ValidationError, |
|||
odoo.exceptions.Warning, |
|||
odoo.exceptions.except_orm</tt></td> |
|||
</tr> |
|||
<tr><td><tt class="docutils literal">sentry_include_context</tt></td> |
|||
<td>If enabled, additional context data will be extracted from current |
|||
HTTP request and user session (if available). This has no effect |
|||
for Cron jobs, as no request/session is available inside a Cron job.</td> |
|||
<td><tt class="docutils literal">True</tt></td> |
|||
</tr> |
|||
<tr><td><tt class="docutils literal">sentry_release</tt></td> |
|||
<td>Explicitly define a version to be sent as the release version to |
|||
Sentry. Useful in conjuntion with Sentry’s “Resolve in the next |
|||
release”-functionality. Also useful if your production deployment |
|||
does not include any Git context from which a commit might be read. |
|||
Overrides <em>sentry_odoo_dir</em>.</td> |
|||
<td> </td> |
|||
</tr> |
|||
<tr><td><tt class="docutils literal">sentry_odoo_dir</tt></td> |
|||
<td>Absolute path to your Odoo installation directory. This is optional |
|||
and will only be used to extract the Odoo Git commit, which will be |
|||
sent to Sentry, to allow to distinguish between Odoo updates. |
|||
Overridden by <em>sentry_release</em></td> |
|||
<td> </td> |
|||
</tr> |
|||
</tbody> |
|||
</table> |
|||
<p>Other <a class="reference external" href="https://docs.sentry.io/platforms/python/configuration/">client arguments</a> can be |
|||
configured by prepending the argument name with <em>sentry_</em> in your Odoo config |
|||
file. Currently supported additional client arguments are: <tt class="docutils literal">with_locals, |
|||
max_breadcrumbs, release, environment, server_name, shutdown_timeout, |
|||
in_app_include, in_app_exclude, default_integrations, dist, sample_rate, |
|||
send_default_pii, http_proxy, https_proxy, request_bodies, debug, |
|||
attach_stacktrace, ca_certs, propagate_traces, traces_sample_rate, |
|||
auto_enabling_integrations</tt>.</p> |
|||
<div class="section" id="example-odoo-configuration"> |
|||
<h1>Example Odoo configuration</h1> |
|||
<p>Below is an example of Odoo configuration file with <em>Odoo Sentry</em> options:</p> |
|||
<pre class="literal-block"> |
|||
[options] |
|||
sentry_dsn = https://<public_key>:<secret_key>@sentry.example.com/<project id> |
|||
sentry_enabled = true |
|||
sentry_logging_level = warn |
|||
sentry_exclude_loggers = werkzeug |
|||
sentry_ignore_exceptions = odoo.exceptions.AccessDenied, |
|||
odoo.exceptions.AccessError,odoo.exceptions.MissingError, |
|||
odoo.exceptions.RedirectWarning,odoo.exceptions.UserError, |
|||
odoo.exceptions.ValidationError,odoo.exceptions.Warning, |
|||
odoo.exceptions.except_orm |
|||
sentry_include_context = true |
|||
sentry_environment = production |
|||
sentry_release = 1.3.2 |
|||
sentry_odoo_dir = /home/odoo/odoo/ |
|||
</pre> |
|||
<p><strong>Table of contents</strong></p> |
|||
<div class="contents local topic" id="contents"> |
|||
<ul class="simple"> |
|||
<li><a class="reference internal" href="#installation" id="id1">Installation</a></li> |
|||
<li><a class="reference internal" href="#configuration" id="id2">Configuration</a></li> |
|||
<li><a class="reference internal" href="#usage" id="id3">Usage</a></li> |
|||
<li><a class="reference internal" href="#known-issues-roadmap" id="id4">Known issues / Roadmap</a></li> |
|||
<li><a class="reference internal" href="#bug-tracker" id="id5">Bug Tracker</a></li> |
|||
<li><a class="reference internal" href="#credits" id="id6">Credits</a><ul> |
|||
<li><a class="reference internal" href="#authors" id="id7">Authors</a></li> |
|||
<li><a class="reference internal" href="#contributors" id="id8">Contributors</a></li> |
|||
<li><a class="reference internal" href="#other-credits" id="id9">Other credits</a></li> |
|||
<li><a class="reference internal" href="#maintainers" id="id10">Maintainers</a></li> |
|||
</ul> |
|||
</li> |
|||
</ul> |
|||
</div> |
|||
<div class="section" id="installation"> |
|||
<h2><a class="toc-backref" href="#id1">Installation</a></h2> |
|||
<p>The module can be installed just like any other Odoo module, by adding the |
|||
module’s directory to Odoo <em>addons_path</em>. In order for the module to correctly |
|||
wrap the Odoo WSGI application, it also needs to be loaded as a server-wide |
|||
module. This can be done with the <tt class="docutils literal">server_wide_modules</tt> parameter in your |
|||
Odoo config file or with the <tt class="docutils literal"><span class="pre">--load</span></tt> command-line parameter.</p> |
|||
<p>This module additionally requires the sentry-sdk Python package to be available on |
|||
the system. It can be installed using pip:</p> |
|||
<pre class="literal-block"> |
|||
pip install sentry-sdk |
|||
</pre> |
|||
</div> |
|||
<div class="section" id="configuration"> |
|||
<h2><a class="toc-backref" href="#id2">Configuration</a></h2> |
|||
<p>The following additional configuration options can be added to your Odoo |
|||
configuration file:</p> |
|||
</div> |
|||
<div class="section" id="usage"> |
|||
<h2><a class="toc-backref" href="#id3">Usage</a></h2> |
|||
<p>Once configured and installed, the module will report any logging event at and |
|||
above the configured Sentry logging level, no additional actions are necessary.</p> |
|||
<a class="reference external image-reference" href="https://runbot.odoo-community.org/runbot/149/10.0"><img alt="Try me on Runbot" src="https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas" /></a> |
|||
</div> |
|||
<div class="section" id="known-issues-roadmap"> |
|||
<h2><a class="toc-backref" href="#id4">Known issues / Roadmap</a></h2> |
|||
<ul class="simple"> |
|||
<li><strong>No database separation</strong> – This module functions by intercepting all Odoo |
|||
logging records in a running Odoo process. This means that once installed in |
|||
one database, it will intercept and report errors for all Odoo databases, |
|||
which are used on that Odoo server.</li> |
|||
<li><strong>Frontend integration</strong> – In the future, it would be nice to add |
|||
Odoo client-side error reporting to this module as well, by integrating |
|||
<a class="reference external" href="https://github.com/getsentry/raven-js">raven-js</a>. Additionally, <a class="reference external" href="https://docs.sentry.io/learn/user-feedback/">Sentry user |
|||
feedback form</a> could be |
|||
integrated into the Odoo client error dialog window to allow users shortly |
|||
describe what they were doing when things went wrong.</li> |
|||
</ul> |
|||
</div> |
|||
<div class="section" id="bug-tracker"> |
|||
<h2><a class="toc-backref" href="#id5">Bug Tracker</a></h2> |
|||
<p>Bugs are tracked on <a class="reference external" href="https://github.com/OCA/server-tools/issues">GitHub Issues</a>. |
|||
In case of trouble, please check there if your issue has already been reported. |
|||
If you spotted it first, help us smashing it by providing a detailed and welcomed |
|||
<a class="reference external" href="https://github.com/OCA/server-tools/issues/new?body=module:%20sentry%0Aversion:%2012.0-migsentrynewapi-fer%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**">feedback</a>.</p> |
|||
<p>Do not contact contributors directly about support or help with technical issues.</p> |
|||
</div> |
|||
<div class="section" id="credits"> |
|||
<h2><a class="toc-backref" href="#id6">Credits</a></h2> |
|||
<div class="section" id="authors"> |
|||
<h3><a class="toc-backref" href="#id7">Authors</a></h3> |
|||
<ul class="simple"> |
|||
<li>Mohammed Barsi</li> |
|||
<li>Versada</li> |
|||
<li>Nicolas JEUDY</li> |
|||
<li>Vauxoo</li> |
|||
</ul> |
|||
</div> |
|||
<div class="section" id="contributors"> |
|||
<h3><a class="toc-backref" href="#id8">Contributors</a></h3> |
|||
<ul class="simple"> |
|||
<li>Mohammed Barsi <<a class="reference external" href="mailto:barsintod@gmail.com">barsintod@gmail.com</a>></li> |
|||
<li>Andrius Preimantas <<a class="reference external" href="mailto:andrius@versada.eu">andrius@versada.eu</a>></li> |
|||
<li>Naglis Jonaitis <<a class="reference external" href="mailto:naglis@versada.eu">naglis@versada.eu</a>></li> |
|||
<li>Atte Isopuro <<a class="reference external" href="mailto:atte.isopuro@avoin.systems">atte.isopuro@avoin.systems</a>></li> |
|||
</ul> |
|||
</div> |
|||
<div class="section" id="other-credits"> |
|||
<h3><a class="toc-backref" href="#id9">Other credits</a></h3> |
|||
<ul class="simple"> |
|||
<li>Vauxoo</li> |
|||
</ul> |
|||
</div> |
|||
<div class="section" id="maintainers"> |
|||
<h3><a class="toc-backref" href="#id10">Maintainers</a></h3> |
|||
<p>This module is maintained by the OCA.</p> |
|||
<a class="reference external image-reference" href="https://odoo-community.org"><img alt="Odoo Community Association" src="https://odoo-community.org/logo.png" /></a> |
|||
<p>OCA, or the Odoo Community Association, is a nonprofit organization whose |
|||
mission is to support the collaborative development of Odoo features and |
|||
promote its widespread use.</p> |
|||
<p>Current <a class="reference external" href="https://odoo-community.org/page/maintainer-role">maintainers</a>:</p> |
|||
<p><a class="reference external" href="https://github.com/barsi"><img alt="barsi" src="https://github.com/barsi.png?size=40px" /></a> <a class="reference external" href="https://github.com/naglis"><img alt="naglis" src="https://github.com/naglis.png?size=40px" /></a> <a class="reference external" href="https://github.com/versada"><img alt="versada" src="https://github.com/versada.png?size=40px" /></a> <a class="reference external" href="https://github.com/moylop260"><img alt="moylop260" src="https://github.com/moylop260.png?size=40px" /></a> <a class="reference external" href="https://github.com/fernandahf"><img alt="fernandahf" src="https://github.com/fernandahf.png?size=40px" /></a></p> |
|||
<p>This module is part of the <a class="reference external" href="https://github.com/OCA/server-tools/tree/12.0-migsentrynewapi-fer/sentry">OCA/server-tools</a> project on GitHub.</p> |
|||
<p>You are welcome to contribute. To learn how please visit <a class="reference external" href="https://odoo-community.org/page/Contribute">https://odoo-community.org/page/Contribute</a>.</p> |
|||
</div> |
|||
</div> |
|||
</div> |
|||
</div> |
|||
</body> |
|||
</html> |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue