Browse Source

[WIP] mis_builder refactoring: cette fois je tiens le bon bout :)

pull/189/head
Stéphane Bidoul 9 years ago
parent
commit
a4193139bc
  1. 483
      mis_builder/models/mis_builder.py
  2. 10
      mis_builder/static/src/xml/mis_widget.xml

483
mis_builder/models/mis_builder.py

@ -1,14 +1,14 @@
# -*- coding: utf-8 -*-
# © 2014-2015 ACSONE SA/NV (<http://acsone.eu>)
# © 2014-2016 ACSONE SA/NV (<http://acsone.eu>)
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
from collections import defaultdict, OrderedDict
from collections import OrderedDict
import datetime
import dateutil
from itertools import izip
import logging
import re
import time
import traceback
import pytz
@ -20,17 +20,11 @@ from .aggregate import _sum, _avg, _min, _max
from .accounting_none import AccountingNone
from openerp.exceptions import UserError
from .simple_array import SimpleArray
from .mis_safe_eval import mis_safe_eval, DataError
_logger = logging.getLogger(__name__)
class DataError(Exception):
def __init__(self, name, msg):
self.name = name
self.msg = msg
class AutoStruct(object):
def __init__(self, **kwargs):
@ -38,133 +32,153 @@ class AutoStruct(object):
setattr(self, k, v)
class KpiMatrix(object):
""" This object holds the computation results in a way
that can be browsed easily for rendering """
class KpiMatrixRow(object):
def __init__(self, env):
self.env = env
# { period: {kpi: vals}
self._kpi_vals = defaultdict(dict)
# { period: {kpi: {account_id: vals}}}
self._kpi_exploded_vals = defaultdict(dict)
# { period: localdict }
self._localdict = {}
# { kpi: set(account_ids) }
self._kpis = OrderedDict()
# { account_id: account name }
self._account_names_by_id = {}
def _get_row_id(self, kpi, account_id=None):
r = kpi.name
if account_id:
r += ':' + str(account_id)
return r
def __init__(self, kpi, account_id=None, parent_row=None):
self.kpi = kpi
self.account_id = account_id
self.description = kpi.description
self.comment = ''
self.parent_row = parent_row
def declare_kpi(self, kpi):
raise RuntimeError("not implemented")
@property
def style(self):
return self.kpi.style
def declare_period(self, period_key, localdict, subkpis,
description, comment):
raise RuntimeError("not implemented")
def iter_cell_tuples(self, cols):
for col in cols:
yield col.get_cell_tuple_for_row(self)
def set_values(self, kpi, period_key, vals):
raise RuntimeError("not implemented")
def iter_cells(self, subcols):
for subcol in subcols:
yield subcol.get_cell_for_row(self)
def set_values_detail_account(self, kpi, period_key, account_id, vals):
raise RuntimeError("not implemented")
def add_comparison(self, period_key_1, period_key_2,
description, comment,
after_period_key):
raise RuntimeError("not implemented")
class KpiMatrixCol(object):
def iter_row_headers(self):
""" Iterate rows headers, top down
def __init__(self, description, comment, locals_dict, subkpis):
self.description = description
self.comment = comment
self.locals_dict = locals_dict
self.colspan = subkpis and len(subkpis) or 1
self._subcols = []
if not subkpis:
subcol = KpiMatrixSubCol(self, '', '', 0)
self._subcols.append(subcol)
else:
for i, subkpi in enumerate(subkpis):
subcol = KpiMatrixSubCol(self, subkpi.description, '', i)
self._subcols.append(subcol)
self._cell_tuples_by_row = {} # {row: (cells tuple)}
yields row_id, parent_row_id=None, row_description, row_comment, \
row_style, kpi
"""
raise RuntimeError("not implemented")
def _set_cell_tuple(self, row, cell_tuple):
self._cell_tuples_by_row[row] = cell_tuple
def iter_col_headers(self):
""" Iterate column headers, left to right
def iter_subcols(self):
return self._subcols
yields col_id, col_description, col_comment, col_span, period_key
"""
raise RuntimeError("not implemented")
def iter_cell_tuples(self):
return self._cells_by_row.values()
def iter_subcol_headers(self):
""" Iterate sub columns headers, left to right
def get_cell_tuple_for_row(self, row):
return self._cell_tuples_by_row.get(row)
yields subcol_id, col_id, subcol_description, subcol_comment
"""
raise RuntimeError("not implemented")
def iter_row_values(self):
""" Iterate row values, left to right
class KpiMatrixSubCol(object):
yields row_id, col_id, val, val_rendered, val_comment,
val_style, subkpi, period_key
"""
raise RuntimeError("not implemented")
def __init__(self, col, description, comment, index=0):
self.col = col
self.description = description
self.comment = comment
self.index = index
def iter_kpi_values(self, period_key):
"""
yields kpi, kpi_values
"""
raise RuntimeError("not implemented")
def iter_cells(self):
for cells in self.col.iter_cell_tuples():
yield cells[self.index]
# TODO FIXME old methods to be removed
def get_cell_for_row(self, row):
cell_tuple = self.col.get_cell_tuple_for_row(row)
return cell_tuple[self.index]
def set_kpi_vals(self, period, kpi, vals):
""" Set the values for a kpi in a period
vals is a list of sub-kpi values.
"""
self._kpi_vals[period][kpi] = vals
if kpi not in self._kpis:
self._kpis[kpi] = set()
class KpiMatrixCell(object):
def set_kpi_exploded_vals(self, period, kpi, account_id, vals):
""" Set the detail values for a kpi in a period for a GL account
def __init__(self, row, subcol,
val, val_rendered, val_comment,
style=None, drilldown_key=None):
self.row = row
self.subcol = subcol
self.val = val
self.val_rendered = val_rendered
self.val_comment = val_comment
self.drilldown_key = None
This is used by the automatic details mechanism.
vals is a list of sub-kpi values.
"""
exploded_vals = self._kpi_exploded_vals[period]
if kpi not in exploded_vals:
exploded_vals[kpi] = {}
exploded_vals[kpi][account_id] = vals
self._kpis[kpi].add(account_id)
class KpiMatrix(object):
def set_localdict(self, period, localdict):
# TODO FIXME to be removed when we have styles
self._localdict[period] = localdict
def __init__(self, env):
# cache language id for faster rendering
lang = env.user.lang or 'en_US'
self.lang = env['res.lang'].search([('code', '=', lang)])
# data structures
self._kpi_rows = OrderedDict() # { kpi: KpiMatrixRow }
self._detail_rows = {} # { kpi: {account_id: KpiMatrixRow} }
self._cols = OrderedDict() # { period_key: KpiMatrixCol }
def get_localdict(self, period):
# TODO FIXME to be removed when we have styles
return self._localdict[period]
def declare_kpi(self, kpi):
self._kpi_rows[kpi] = KpiMatrixRow(kpi)
self._detail_rows[kpi] = {}
def iter_kpi_vals(self, period):
""" Iterate kpi values, including auto-expanded details by account
def declare_period(self, period_key, description, comment,
locals_dict, subkpis):
self._cols[period_key] = KpiMatrixCol(description, comment,
locals_dict, subkpis)
It yields, in no specific order:
* kpi technical name
* kpi object
* subkpi values tuple
"""
for kpi, vals in self._kpi_vals[period].iteritems():
yield kpi.name, kpi, vals
kpi_exploded_vals = self._kpi_exploded_vals[period]
if kpi not in kpi_exploded_vals:
continue
for account_id, account_id_vals in \
kpi_exploded_vals[kpi].iteritems():
yield "%s:%s" % (kpi.name, account_id), kpi, account_id_vals
def set_values(self, kpi, period_key, vals):
self.set_values_detail_account(kpi, period_key, None, vals)
def set_values_detail_account(self, kpi, period_key, account_id, vals):
if not account_id:
row = self._kpi_rows[kpi]
else:
kpi_row = self._kpi_rows[kpi]
row = KpiMatrixRow(kpi, account_id, parent_row=kpi_row)
self._detail_rows[kpi][account_id] = row
col = self._cols[period_key]
cell_tuple = []
assert len(vals) == col.colspan
for val, subcol in izip(vals, col.iter_subcols()):
if isinstance(val, DataError):
val_rendered = val.name
val_comment = val.msg
else:
val_rendered = kpi.render(self.lang, val)
val_comment = '' # TODO FIXME get subkpi expression
# TODO style
# TODO drilldown_key
cell = KpiMatrixCell(row, subcol, val, val_rendered, val_comment)
cell_tuple.append(cell)
col._set_cell_tuple(row, cell_tuple)
def iter_rows(self):
for kpi_row in self._kpi_rows.values():
yield kpi_row
# TODO FIXME sort detail rows
for detail_row in self._detail_rows[kpi_row.kpi].values():
yield detail_row
def iter_cols(self):
return self._cols.values()
def iter_subcols(self):
for col in self.iter_cols():
for subcol in col.iter_subcols():
yield subcol
def iter_kpis(self):
class old_KpiMatrix(object):
def __iter_kpis(self):
""" Iterate kpis, including auto-expanded details by accounts
It yields, in display order:
@ -178,7 +192,7 @@ class KpiMatrix(object):
yield "%s:%s" % (kpi.name, account_id), \
self.get_account_name(account_id), kpi
def get_exploded_account_ids(self):
def __get_exploded_account_ids(self):
""" Get the list of auto-expanded account ids
It returns the complete list, across all periods and kpis.
@ -190,7 +204,7 @@ class KpiMatrix(object):
res.update(account_ids)
return list(res)
def load_account_names(self, account_obj):
def __load_account_names(self, account_obj):
""" Load account names for all exploded account ids
This method must be called after setting all kpi values
@ -201,7 +215,7 @@ class KpiMatrix(object):
self._account_names_by_id = {a.id: u"{} {}".format(a.code, a.name)
for a in account_data}
def get_account_name(self, account_id):
def __get_account_name(self, account_id):
""" Get account display name from it's id
This method must be called after loading account names with
@ -321,7 +335,7 @@ class MisReportKpi(models.Model):
expression.subkpi_id.name, expression.name))
else:
l.append(
expression.name)
expression.name or 'AccountingNone')
kpi.expression = ',\n'.join(l)
@api.multi
@ -380,21 +394,21 @@ class MisReportKpi(models.Model):
self.divider = ''
self.dp = 0
def render(self, lang_id, value):
def render(self, lang, value):
""" render a KPI value as a unicode string, ready for display """
assert len(self) == 1
if value is None or value is AccountingNone:
return ''
elif self.type == 'num':
return self._render_num(lang_id, value, self.divider,
return self._render_num(lang, value, self.divider,
self.dp, self.prefix, self.suffix)
elif self.type == 'pct':
return self._render_num(lang_id, value, 0.01,
return self._render_num(lang, value, 0.01,
self.dp, '', '%')
else:
return unicode(value) # noqa - silence python3 error
def render_comparison(self, lang_id, value, base_value,
def render_comparison(self, lang, value, base_value,
average_value, average_base_value):
""" render the comparison of two KPI values, ready for display
@ -409,7 +423,7 @@ class MisReportKpi(models.Model):
delta = value - base_value
if delta and round(delta, self.dp) != 0:
return self._render_num(
lang_id,
lang,
delta,
0.01, self.dp, '', _('pp'),
sign='+')
@ -422,7 +436,7 @@ class MisReportKpi(models.Model):
delta = value - base_value
if delta and round(delta, self.dp) != 0:
return self._render_num(
lang_id,
lang,
delta,
self.divider, self.dp, self.prefix, self.suffix,
sign='+')
@ -431,13 +445,13 @@ class MisReportKpi(models.Model):
delta = (value - base_value) / abs(base_value)
if delta and round(delta, self.dp) != 0:
return self._render_num(
lang_id,
lang,
delta,
0.01, self.dp, '', '%',
sign='+')
return ''
def _render_num(self, lang_id, value, divider,
def _render_num(self, lang, value, divider,
dp, prefix, suffix, sign='-'):
divider_label = _get_selection_label(
self._columns['divider'].selection, divider)
@ -445,11 +459,11 @@ class MisReportKpi(models.Model):
divider_label = ''
# format number following user language
value = round(value / float(divider or 1), dp) or 0
value = self.env['res.lang'].browse(lang_id).format(
value = lang.format(
'%%%s.%df' % (sign, dp),
value,
grouping=True)
value = u'%s\N{NARROW NO-BREAK SPACE}%s\N{NO-BREAK SPACE}%s%s' % \
value = u'%s\N{NO-BREAK SPACE}%s\N{NO-BREAK SPACE}%s%s' % \
(prefix or '', value, divider_label, suffix or '')
value = value.replace('-', u'\N{NON-BREAKING HYPHEN}')
return value
@ -512,9 +526,15 @@ class MisReportKpiExpression(models.Model):
readonly=True)
name = fields.Char(string='Expression')
kpi_id = fields.Many2one('mis.report.kpi')
# TODO FIXME set readonly=True when onchange('subkpi_ids') below works
subkpi_id = fields.Many2one(
'mis.report.subkpi',
readonly=True)
readonly=False)
_sql_constraints = [
('subkpi_kpi_unique', 'unique(subkpi_id, kpi_id)',
'Sub KPI must be used once and only once for each KPI'),
]
class MisReportQuery(models.Model):
@ -589,10 +609,33 @@ class MisReport(models.Model):
kpi_ids = fields.One2many('mis.report.kpi', 'report_id',
string='KPI\'s',
copy=True)
subkpi_ids = fields.One2many(
'mis.report.subkpi',
'report_id',
string="Sub KPI")
subkpi_ids = fields.One2many('mis.report.subkpi', 'report_id',
string="Sub KPI",
copy=True)
@api.onchange('subkpi_ids')
def _on_change_subkpi_ids(self):
""" Update kpi expressions when subkpis change on the report,
so the list of kpi expressions is always up-to-date """
for kpi in self.kpi_ids:
if not kpi.multi:
continue
new_subkpis = set([subkpi for subkpi in self.subkpi_ids])
expressions = []
for expression in kpi.expression_ids:
assert expression.subkpi_id # must be true if kpi is multi
if expression.subkpi_id not in self.subkpi_ids:
expressions.append((2, expression.id, None)) # remove
else:
new_subkpis.remove(expression.subkpi_id) # no change
for subkpi in new_subkpis:
# TODO FIXME this does not work, while the remove above works
expressions.append((0, None, {
'name': False,
'subkpi_id': subkpi.id,
})) # add empty expressions for new subkpis
if expressions:
kpi.expressions_ids = expressions
@api.multi
def get_wizard_report_action(self):
@ -632,7 +675,8 @@ class MisReport(models.Model):
self.ensure_one()
aep = AEP(self.env)
for kpi in self.kpi_ids:
aep.parse_expr(kpi.expression)
for expression in kpi.expression_ids:
aep.parse_expr(expression.name)
aep.done_parsing(company)
return aep
@ -694,8 +738,9 @@ class MisReport(models.Model):
return res
@api.multi
def _compute_period(self, kpi_matrix, period_key,
lang_id, aep,
def _compute_period(self, kpi_matrix,
period_key, period_description, period_comment,
aep,
date_from, date_to,
target_move,
company,
@ -706,7 +751,6 @@ class MisReport(models.Model):
:param kpi_matrix: the KpiMatrix object to be populated
:param period_key: the period key to use when populating the KpiMatrix
:param lang_id: id of a res.lang object
:param aep: an AccountingExpressionProcessor instance created
using _prepare_aep()
:param date_from, date_to: the starting and ending date
@ -720,33 +764,23 @@ class MisReport(models.Model):
query argument and returns a
domain compatible with the query
underlying model
For each kpi, it calls set_kpi_vals and set_kpi_exploded_vals
with vals being a tuple with the evaluation
result for sub-kpis, or a DataError object if the evaluation failed.
When done, it also calls set_localdict to store the local values
that served for the computation of the period.
"""
self.ensure_one()
localdict = {
locals_dict = {
'sum': _sum,
'min': _min,
'max': _max,
'len': len,
'avg': _avg,
'AccountingNone': AccountingNone,
'SimpleArray': SimpleArray,
}
# fetch non-accounting queries
localdict.update(self._fetch_queries(
locals_dict.update(self._fetch_queries(
date_from, date_to, get_additional_query_filter))
# prepare the period in the kpi matrix
kpi_matrix.declare_period(period_key, localdict)
# use AEP to do the accounting queries
additional_move_line_filter = None
if get_additional_move_line_filter:
@ -756,83 +790,71 @@ class MisReport(models.Model):
target_move,
additional_move_line_filter)
if subkpis_filter:
subkpis = [subkpi for subkpi in self.subkpi_ids
if subkpi in subkpis_filter]
else:
subkpis = self.subkpi_ids
kpi_matrix.declare_period(period_key,
period_description, period_comment,
locals_dict, subkpis)
compute_queue = self.kpi_ids
recompute_queue = []
while True:
for kpi in compute_queue:
vals = []
has_error = False
# build the list of expressions for this kpi
expressions = []
for expression in kpi.expression_ids:
if expression.subkpi_id and \
subkpis_filter and \
expression.subkpi_id not in subkpis_filter:
continue
try:
kpi_eval_expression = aep.replace_expr(expression.name)
vals.append(safe_eval(kpi_eval_expression, localdict))
except ZeroDivisionError:
has_error = True
vals.append(DataError(
'#DIV/0',
'\n\n%s' % (traceback.format_exc(),)))
except (NameError, ValueError):
has_error = True
recompute_queue.append(kpi)
vals.append(DataError(
'#ERR',
'\n\n%s' % (traceback.format_exc(),)))
except:
has_error = True
vals.append(DataError(
'#ERR',
'\n\n%s' % (traceback.format_exc(),)))
if len(vals) == 1 and isinstance(vals[0], SimpleArray):
vals = vals[0]
else:
vals = SimpleArray(vals)
expressions.append(expression.name)
kpi_matrix.set_kpi_vals(period_key, kpi, vals)
if has_error:
continue
vals = []
try:
for expression in expressions:
replaced_expr = aep.replace_expr(expression)
vals.append(
mis_safe_eval(replaced_expr, locals_dict))
except NameError:
recompute_queue.append(kpi)
break
else:
# no error, set it in locals_dict so it can be used
# in computing other kpis
if len(expressions) == 1:
locals_dict[kpi.name] = vals[0]
else:
locals_dict[kpi.name] = SimpleArray(vals)
# no error, set it in localdict so it can be used
# in computing other kpis
localdict[kpi.name] = vals
kpi_matrix.set_values(kpi, period_key, vals)
# TODO FIXME handle exceptions
if not kpi.auto_expand_accounts:
continue
expr = []
for expression in kpi.expression_ids:
if expression.subkpi_id and \
subkpis_filter and \
expression.subkpi_id not in subkpis_filter:
continue
expr.append(expression.name)
expr = ', '.join(expr) # tuple
for account_id, replaced_expr in \
aep.replace_expr_by_account_id(expr):
account_id_vals = safe_eval(replaced_expr, localdict)
kpi_matrix.set_kpi_exploded_vals(kpi_matrix_period, kpi,
account_id,
account_id_vals)
for account_id, replaced_exprs in \
aep.replace_exprs_by_account_id(expressions):
account_id_vals = []
for replaced_expr in replaced_exprs:
account_id_vals.append(
mis_safe_eval(replaced_expr, locals_dict))
kpi_matrix.set_values_detail_account(
kpi, period_key, account_id, account_id_vals)
if len(recompute_queue) == 0:
# nothing to recompute, we are done
break
if len(recompute_queue) == len(compute_queue):
# could not compute anything in this iteration
# (ie real Value errors or cyclic dependency)
# (ie real Name errors or cyclic dependency)
# so we stop trying
break
# try again
compute_queue = recompute_queue
recompute_queue = []
kpi_matrix.set_localdict(period_key, localdict)
class MisReportInstancePeriod(models.Model):
""" A MIS report instance has the logic to compute
@ -1038,12 +1060,13 @@ class MisReportInstancePeriod(models.Model):
mis.report.instance.period is going to do something
useful in this kpi
"""
# TODO FIXME remove this method
self.ensure_one()
# first invoke the compute method on the mis report template
# passing it all the information regarding period and filters
self.report_instance_id.report_id._compute_period(
kpi_matrix, self,
lang_id, aep,
aep,
self.date_from, self.date_to,
self.report_instance_id.target_move,
self.report_instance_id.company_id,
@ -1061,7 +1084,7 @@ class MisReportInstancePeriod(models.Model):
kpi_style = None
if kpi.style_expression:
style_name = safe_eval(kpi.style_expression,
kpi_matrix.get_localdict(self))
kpi_matrix.get_locals_dict(self))
styles = mis_report_kpi_style.search(
[('name', '=', style_name)])
kpi_style = styles and styles[0]
@ -1302,6 +1325,66 @@ class MisReportInstance(models.Model):
def compute(self):
self.ensure_one()
aep = self.report_id._prepare_aep(self.company_id)
kpi_matrix = self.report_id._prepare_kpi_matrix()
for period in self.period_ids:
self.report_id._compute_period(
kpi_matrix,
period.id,
'period name', # TODO FIXME
'period comment', # TODO FIXME
aep,
period.date_from,
period.date_to,
self.target_move,
self.company_id,
period.subkpi_ids,
period._get_additional_move_line_filter,
period._get_additional_query_filter)
header = [{'cols': []}, {'cols': []}]
for col in kpi_matrix.iter_cols():
header[0]['cols'].append({
'description': col.description,
'comment': col.comment,
'colspan': col.colspan,
})
for subcol in col.iter_subcols():
header[1]['cols'].append({
'description': subcol.description,
'comment': subcol.comment,
'colspan': 1,
})
content = []
for row in kpi_matrix.iter_rows():
row_data = {
'row_id': id(row),
'parent_row_id': row.parent_row and id(row.parent_row) or None,
'description': row.description,
'comment': row.comment,
'style': row.style and row.style.to_css_style() or '',
'cols': []
}
for cell in row.iter_cells(kpi_matrix.iter_subcols()):
row_data['cols'].append({
'val': (cell.val
if cell.val is not AccountingNone else None),
'val_r': cell.val_rendered,
'val_c': cell.val_comment,
# TODO FIXME style
# TODO FIXME drilldown
})
content.append(row_data)
return {
'header': header,
'content': content,
}
@api.multi
def old_compute(self):
self.ensure_one()
aep = self.report_id._prepare_aep(self.company_id)
# fetch user language only once
# TODO: is this necessary?

10
mis_builder/static/src/xml/mis_widget.xml

@ -13,16 +13,10 @@
<t t-foreach="widget.mis_report_data.header" t-as="h">
<tr class="oe_list_header_columns">
<th class="oe_list_header_char">
<div>
<t t-esc="h_value.kpi_name"/>
</div>
</th>
<th t-foreach="h_value.cols" t-as="col" class="oe_list_header_char mis_builder_ralign" t-att-colspan="col.colspan">
<div>
<t t-esc="col.name"/>
</div>
<div>
<t t-esc="col.date"/>
<t t-esc="col.description"/>
</div>
</th>
</tr>
@ -32,7 +26,7 @@
<tr t-foreach="widget.mis_report_data.content" t-as="c">
<td t-att="{'style': c_value.default_style}">
<div>
<t t-esc="c_value.kpi_name"/>
<t t-esc="c_value.description"/>
</div>
</td>
<t t-foreach="c_value.cols" t-as="value">

Loading…
Cancel
Save