mirror of
https://github.com/odoo/runbot.git
synced 2025-03-15 23:45:44 +07:00
[REF] runbot: adapt statistic models
Initial model was a (build_id, key, value) where key is in fact a two
part information: `category.key` (where key is usually a module)
This means that for each module, we will have one entry per
modules*category.
We have between 200 and 400 modules per build * 4 keys -> around 1000
entries per build.
The hudge amount of total entries lead to a fast overflow of the table
sequence + this create important indexes.
Also, most of the time, the js will manage the display of stats meaning
that python will transform
(build_id, category.key, value)
into
(build_id, {key: value}) for one category
A new model makes use of a json field to store values for different
modules in a json dict, on entry per category*build. (4 entry per build)
The table will be renamed and migrate later
This commit is contained in:
Xavier-Do
parent
87366de213
commit
d466887f08
@ -453,10 +453,9 @@ class Runbot(Controller):
|
||||
return request.not_found()
|
||||
|
||||
build_stats = defaultdict(dict)
|
||||
for stat in build.stat_ids.filtered(lambda rec: '.' in rec.key).sorted(key=lambda rec: rec.value, reverse=True):
|
||||
category, module = stat.key.split('.', maxsplit=1)
|
||||
value = int(stat.value) if stat.value == int(stat.value) else stat.value
|
||||
build_stats[category].update({module: value})
|
||||
for stat in build.stat_ids:
|
||||
for module, value in sorted(stat.values.items(), key=lambda item: item[1], reverse=True):
|
||||
build_stats[stat.category][module] = value
|
||||
|
||||
context = {
|
||||
'build': build,
|
||||
@ -500,10 +499,11 @@ class Runbot(Controller):
|
||||
builds = builds.search([('id', 'child_of', builds.ids)])
|
||||
|
||||
parents = {b.id: b.top_parent.id for b in builds.with_context(prefetch_fields=False)}
|
||||
request.env.cr.execute("SELECT build_id, key, value FROM runbot_build_stat WHERE build_id IN %s AND key like %s", [tuple(builds.ids), '%s.%%' % key_category]) # read manually is way faster than using orm
|
||||
request.env.cr.execute("SELECT build_id, values FROM runbot_build_stat WHERE build_id IN %s AND category = %s", [tuple(builds.ids), key_category]) # read manually is way faster than using orm
|
||||
res = {}
|
||||
for (builds_id, key, value) in request.env.cr.fetchall():
|
||||
res.setdefault(parents[builds_id], {})[key.split('.', 1)[1]] = value
|
||||
for (build_id, values) in request.env.cr.fetchall():
|
||||
res.setdefault(parents[build_id], {}).update(values)
|
||||
# we need to update here to manage the post install case: we want to combine stats from all post_install childrens.
|
||||
return res
|
||||
|
||||
@route(['/runbot/stats/<model("runbot.bundle"):bundle>/<model("runbot.trigger"):trigger>'], type='http', auth="public", website=True, sitemap=False)
|
||||
|
||||
@ -1035,8 +1035,17 @@ class ConfigStep(models.Model):
|
||||
regex_ids = self.build_stat_regex_ids
|
||||
if not regex_ids:
|
||||
regex_ids = regex_ids.search([('generic', '=', True)])
|
||||
key_values = regex_ids._find_in_file(log_path)
|
||||
self.env['runbot.build.stat']._write_key_values(build, self, key_values)
|
||||
stats_per_regex = regex_ids._find_in_file(log_path)
|
||||
if stats_per_regex:
|
||||
build_stats = [
|
||||
{
|
||||
'config_step_id': self.id,
|
||||
'build_id': build.id,
|
||||
'category': category,
|
||||
'values': values,
|
||||
} for category, values in stats_per_regex.items()
|
||||
]
|
||||
self.env['runbot.build.stat'].create(build_stats)
|
||||
except Exception as e:
|
||||
message = '**An error occured while computing statistics of %s:**\n`%s`' % (build.job, str(e).replace('\\n', '\n').replace("\\'", "'"))
|
||||
_logger.exception(message)
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import logging
|
||||
|
||||
from odoo import models, fields, api, tools
|
||||
from ..fields import JsonDictField
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
@ -13,7 +14,7 @@ class BuildStat(models.Model):
|
||||
_sql_constraints = [
|
||||
(
|
||||
"build_config_key_unique",
|
||||
"unique (build_id, config_step_id,key)",
|
||||
"unique (build_id, config_step_id, category)",
|
||||
"Build stats must be unique for the same build step",
|
||||
)
|
||||
]
|
||||
@ -22,89 +23,5 @@ class BuildStat(models.Model):
|
||||
config_step_id = fields.Many2one(
|
||||
"runbot.build.config.step", "Step", ondelete="cascade"
|
||||
)
|
||||
key = fields.Char("key", index=True)
|
||||
value = fields.Float("Value")
|
||||
|
||||
@api.model
|
||||
def _write_key_values(self, build, config_step, key_values):
|
||||
if not key_values:
|
||||
return self
|
||||
build_stats = [
|
||||
{
|
||||
"config_step_id": config_step.id,
|
||||
"build_id": build.id,
|
||||
"key": k,
|
||||
"value": v,
|
||||
}
|
||||
for k, v in key_values.items()
|
||||
]
|
||||
return self.create(build_stats)
|
||||
|
||||
|
||||
class RunbotBuildStatSql(models.Model):
|
||||
|
||||
_name = "runbot.build.stat.sql"
|
||||
_description = "Build stat sql view"
|
||||
_auto = False
|
||||
|
||||
bundle_id = fields.Many2one("runbot.bundle", string="Bundle", readonly=True)
|
||||
bundle_name = fields.Char(string="Bundle name", readonly=True)
|
||||
bundle_sticky = fields.Boolean(string="Sticky", readonly=True)
|
||||
batch_id = fields.Many2one("runbot.bundle", string="Batch", readonly=True)
|
||||
trigger_id = fields.Many2one("runbot.trigger", string="Trigger", readonly=True)
|
||||
trigger_name = fields.Char(string="Trigger name", readonly=True)
|
||||
|
||||
stat_id = fields.Many2one("runbot.build.stat", string="Stat", readonly=True)
|
||||
key = fields.Char("Key", readonly=True)
|
||||
value = fields.Float("Value", readonly=True)
|
||||
|
||||
config_step_id = fields.Many2one(
|
||||
"runbot.build.config.step", string="Config Step", readonly=True
|
||||
)
|
||||
config_step_name = fields.Char(string="Config Step name", readonly=True)
|
||||
|
||||
build_id = fields.Many2one("runbot.build", string="Build", readonly=True)
|
||||
build_config_id = fields.Many2one("runbot.build.config", string="Config", readonly=True)
|
||||
build_parent_path = fields.Char('Build Parent path')
|
||||
build_host = fields.Char(string="Host", readonly=True)
|
||||
|
||||
def init(self):
|
||||
""" Create SQL view for build stat """
|
||||
tools.drop_view_if_exists(self._cr, "runbot_build_stat_sql")
|
||||
self._cr.execute(
|
||||
""" CREATE OR REPLACE VIEW runbot_build_stat_sql AS (
|
||||
SELECT
|
||||
(stat.id::bigint*(2^32)+bun.id::bigint) AS id,
|
||||
stat.id AS stat_id,
|
||||
stat.key AS key,
|
||||
stat.value AS value,
|
||||
step.id AS config_step_id,
|
||||
step.name AS config_step_name,
|
||||
bu.id AS build_id,
|
||||
bp.config_id AS build_config_id,
|
||||
bu.parent_path AS build_parent_path,
|
||||
bu.host AS build_host,
|
||||
bun.id AS bundle_id,
|
||||
bun.name AS bundle_name,
|
||||
bun.sticky AS bundle_sticky,
|
||||
ba.id AS batch_id,
|
||||
tr.id AS trigger_id,
|
||||
tr.name AS trigger_name
|
||||
FROM
|
||||
runbot_build_stat AS stat
|
||||
JOIN
|
||||
runbot_build_config_step step ON stat.config_step_id = step.id
|
||||
JOIN
|
||||
runbot_build bu ON bu.id = stat.build_id
|
||||
JOIN
|
||||
runbot_build_params bp ON bp.id =bu.params_id
|
||||
JOIN
|
||||
runbot_batch_slot bas ON bas.build_id = stat.build_id
|
||||
JOIN
|
||||
runbot_trigger tr ON tr.id = bas.trigger_id
|
||||
JOIN
|
||||
runbot_batch ba ON ba.id = bas.batch_id
|
||||
JOIN
|
||||
runbot_bundle bun ON bun.id = ba.bundle_id
|
||||
)"""
|
||||
)
|
||||
category = fields.Char("Category", index=True)
|
||||
values = JsonDictField("Value")
|
||||
|
||||
@ -50,10 +50,11 @@ class BuildStatRegex(models.Model):
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
return {}
|
||||
key_values = {}
|
||||
stats_matches = {}
|
||||
with open(file_path, "r") as log_file:
|
||||
data = log_file.read()
|
||||
for build_stat_regex in self:
|
||||
current_stat_matches = {}
|
||||
for match in re.finditer(build_stat_regex.regex, data):
|
||||
group_dict = match.groupdict()
|
||||
try:
|
||||
@ -64,10 +65,6 @@ class BuildStatRegex(models.Model):
|
||||
group_dict.get("value"), build_stat_regex.regex
|
||||
)
|
||||
continue
|
||||
key = (
|
||||
"%s.%s" % (build_stat_regex.name, group_dict["key"])
|
||||
if "key" in group_dict
|
||||
else build_stat_regex.name
|
||||
)
|
||||
key_values[key] = value
|
||||
return key_values
|
||||
current_stat_matches[group_dict.get('key', 'value')] = value
|
||||
stats_matches[build_stat_regex.name] = current_stat_matches
|
||||
return stats_matches
|
||||
|
||||
@ -47,13 +47,9 @@ access_runbot_repo_referencetime,runbot_repo_referencetime,runbot.model_runbot_r
|
||||
access_runbot_build_stat_user,runbot_build_stat_user,runbot.model_runbot_build_stat,group_user,1,0,0,0
|
||||
access_runbot_build_stat_admin,runbot_build_stat_admin,runbot.model_runbot_build_stat,runbot.group_runbot_admin,1,1,1,1
|
||||
|
||||
access_runbot_build_stat_sql_user,runbot_build_stat_sql_user,runbot.model_runbot_build_stat_sql,group_user,1,0,0,0
|
||||
access_runbot_build_stat_sql_admin,runbot_build_stat_sql_admin,runbot.model_runbot_build_stat_sql,runbot.group_runbot_admin,1,0,0,0
|
||||
|
||||
access_runbot_build_stat_regex_user,access_runbot_build_stat_regex_user,runbot.model_runbot_build_stat_regex,runbot.group_user,1,0,0,0
|
||||
access_runbot_build_stat_regex_admin,access_runbot_build_stat_regex_admin,runbot.model_runbot_build_stat_regex,runbot.group_runbot_admin,1,1,1,1
|
||||
|
||||
|
||||
access_runbot_trigger_user,access_runbot_trigger_user,runbot.model_runbot_trigger,runbot.group_user,1,0,0,0
|
||||
access_runbot_trigger_runbot_admin,access_runbot_trigger_runbot_admin,runbot.model_runbot_trigger,runbot.group_runbot_admin,1,1,1,1
|
||||
|
||||
|
||||
|
@ -47,6 +47,7 @@ class TestBuildStatRegex(RunbotCase):
|
||||
|
||||
def test_build_stat_regex_find_in_file(self):
|
||||
|
||||
max_id = self.BuildStat.search([], order="id desc", limit=1).id or 0
|
||||
file_content = """foo bar
|
||||
2020-03-02 22:06:58,391 17 INFO xxx odoo.modules.module: odoo.addons.website_blog.tests.test_ui tested in 10.35s, 2501 queries
|
||||
some garbage
|
||||
@ -58,24 +59,30 @@ nothing to see here
|
||||
)
|
||||
with patch("builtins.open", mock_open(read_data=file_content)):
|
||||
self.config_step._make_stats(self.build)
|
||||
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_blog.tests.test_ui'), ('value', '=', 2501.0)]), 1)
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_event.tests.test_ui'), ('value', '=', 2435.0)]), 1)
|
||||
self.assertEqual(
|
||||
dict(self.BuildStat.search([('category', '=', 'query_count'), ('id', '>', max_id)]).values),
|
||||
{
|
||||
'website_event.tests.test_ui': 2435.0,
|
||||
'website_blog.tests.test_ui': 2501.0
|
||||
}
|
||||
)
|
||||
|
||||
# Check unicity
|
||||
with self.assertRaises(IntegrityError):
|
||||
with mute_logger("odoo.sql_db"):
|
||||
with self.cr.savepoint(): # needed to continue tests
|
||||
self.env["runbot.build.stat"]._write_key_values(
|
||||
self.build, self.config_step, {'query_count.website_event.tests.test_ui': 2435}
|
||||
)
|
||||
self.env["runbot.build.stat"].create({
|
||||
'build_id': self.build.id,
|
||||
'config_step_id': self.config_step.id,
|
||||
'category': 'query_count',
|
||||
'values': {'website_event.tests.test_ui': 2435},
|
||||
}
|
||||
|
||||
# minimal test for RunbotBuildStatSql model
|
||||
# self.assertEqual(self.env['runbot.build.stat.sql'].search_count([('build_id', '=', self.build.id)]), 2)
|
||||
# TODO FIXME
|
||||
)
|
||||
|
||||
def test_build_stat_regex_generic(self):
|
||||
""" test that regex are not used when generic is False and that _make_stats use all genreic regex if there are no regex on step """
|
||||
max_id = self.BuildStat.search([], order="id desc", limit=1).id or 0
|
||||
file_content = """foo bar
|
||||
odoo.addons.foobar tested in 2s, 25 queries
|
||||
useless 10
|
||||
@ -96,12 +103,12 @@ chocolate 15
|
||||
with patch("builtins.open", mock_open(read_data=file_content)):
|
||||
self.config_step._make_stats(self.build)
|
||||
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.foobar'), ('value', '=', 25.0)]), 0)
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'useless_count.useless'), ('value', '=', 10.0)]), 0)
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'chocolate_count.chocolate'), ('value', '=', 15.0)]), 1)
|
||||
self.assertEqual(self.BuildStat.search_count([('category', '=', 'query_count'), ('id', '>', max_id)]), 0)
|
||||
self.assertEqual(self.BuildStat.search_count([('category', '=', 'useless_count'), ('id', '>', max_id)]), 0)
|
||||
self.assertEqual(dict(self.BuildStat.search([('category', '=', 'chocolate_count'), ('id', '>', max_id)]).values), {'chocolate': 15.0})
|
||||
|
||||
def test_build_stat_regex_find_in_file_perf(self):
|
||||
|
||||
max_id = self.BuildStat.search([], order="id desc", limit=1).id or 0
|
||||
noise_lines = """2020-03-17 13:26:15,472 2376 INFO runbottest odoo.modules.loading: loading runbot/views/build_views.xml
|
||||
2020-03-10 22:58:34,472 17 INFO 1709329-master-9938b2-all_no_autotag werkzeug: 127.0.0.1 - - [10/Mar/2020 22:58:34] "POST /mail/read_followers HTTP/1.1" 200 - 13 0.004 0.009
|
||||
2020-03-10 22:58:30,137 17 INFO ? werkzeug: 127.0.0.1 - - [10/Mar/2020 22:58:30] "GET /website/static/src/xml/website.editor.xml HTTP/1.1" 200 - - - -
|
||||
@ -125,5 +132,10 @@ chocolate 15
|
||||
with patch("builtins.open", mock_open(read_data=log_data)):
|
||||
self.config_step._make_stats(self.build)
|
||||
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_blog.tests.test_ui'), ('value', '=', 2501.0)]), 1)
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_event.tests.test_ui'), ('value', '=', 2435.0)]), 1)
|
||||
self.assertEqual(
|
||||
dict(self.BuildStat.search([('category', '=', 'query_count'), ('id', '>', max_id)]).values),
|
||||
{
|
||||
'website_event.tests.test_ui': 2435.0,
|
||||
'website_blog.tests.test_ui': 2501.0
|
||||
}
|
||||
)
|
||||
|
||||
@ -86,17 +86,6 @@
|
||||
<field name="gc_delay"/>
|
||||
</group>
|
||||
</sheet>
|
||||
<sheet>
|
||||
<group>
|
||||
<field name="stat_ids">
|
||||
<tree>
|
||||
<field name="config_step_id"/>
|
||||
<field name="key"/>
|
||||
<field name="value"/>
|
||||
</tree>
|
||||
</field>
|
||||
</group>
|
||||
</sheet>
|
||||
</form>
|
||||
</field>
|
||||
</record>
|
||||
|
||||
@ -9,7 +9,6 @@
|
||||
<menuitem id="menu_bundle_batch" action="action_bundle_batch" parent="menu_objects" sequence="25" />
|
||||
<menuitem name="Branches" id="runbot_menu_branch_tree" parent="menu_objects" sequence="40" action="open_view_branch_tree"/>
|
||||
<menuitem name="Commits" id="runbot_menu_commit_tree" parent="menu_objects" sequence="50" action="open_view_commit_tree"/>
|
||||
<menuitem name="Statistics" id="runbot_menu_stat_tree" parent="menu_objects" sequence="60" action="open_view_stat_sql_tree"/>
|
||||
<menuitem id="runbot_menu_bundle_custom_trigger" parent="menu_objects" sequence="70" action="action_bundle_custom_trigger"/>
|
||||
|
||||
<menuitem name="Hosts" id="runbot_menu_host_tree" parent="runbot_menu_root" sequence="300" action="open_view_host_tree"/>
|
||||
|
||||
@ -1,29 +1,5 @@
|
||||
<odoo>
|
||||
<data>
|
||||
|
||||
<record id="view_stat_sql_tree" model="ir.ui.view">
|
||||
<field name="name">runbot.stat.sql.tree</field>
|
||||
<field name="model">runbot.build.stat.sql</field>
|
||||
<field name="arch" type="xml">
|
||||
<tree string="Statistics">
|
||||
<field name="key"/>
|
||||
<field name="value"/>
|
||||
<field name="config_step_id"/>
|
||||
<field name="build_id"/>
|
||||
<field name="bundle_id"/>
|
||||
<field name="batch_id"/>
|
||||
<field name="trigger_id"/>
|
||||
<field name="stat_id"/>
|
||||
</tree>
|
||||
</field>
|
||||
</record>
|
||||
|
||||
<record id="open_view_stat_sql_tree" model="ir.actions.act_window">
|
||||
<field name="name">Statistics</field>
|
||||
<field name="res_model">runbot.build.stat.sql</field>
|
||||
<field name="view_mode">tree,graph,pivot</field>
|
||||
</record>
|
||||
|
||||
<record id="build_stat_regex_form" model="ir.ui.view">
|
||||
<field name="name">runbot.build.stat.regex.form</field>
|
||||
<field name="model">runbot.build.stat.regex</field>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user