diff --git a/runbot/controllers/frontend.py b/runbot/controllers/frontend.py index 75310d4f..557eb3b6 100644 --- a/runbot/controllers/frontend.py +++ b/runbot/controllers/frontend.py @@ -453,10 +453,9 @@ class Runbot(Controller): return request.not_found() build_stats = defaultdict(dict) - for stat in build.stat_ids.filtered(lambda rec: '.' in rec.key).sorted(key=lambda rec: rec.value, reverse=True): - category, module = stat.key.split('.', maxsplit=1) - value = int(stat.value) if stat.value == int(stat.value) else stat.value - build_stats[category].update({module: value}) + for stat in build.stat_ids: + for module, value in sorted(stat.values.items(), key=lambda item: item[1], reverse=True): + build_stats[stat.category][module] = value context = { 'build': build, @@ -500,10 +499,11 @@ class Runbot(Controller): builds = builds.search([('id', 'child_of', builds.ids)]) parents = {b.id: b.top_parent.id for b in builds.with_context(prefetch_fields=False)} - request.env.cr.execute("SELECT build_id, key, value FROM runbot_build_stat WHERE build_id IN %s AND key like %s", [tuple(builds.ids), '%s.%%' % key_category]) # read manually is way faster than using orm + request.env.cr.execute("SELECT build_id, values FROM runbot_build_stat WHERE build_id IN %s AND category = %s", [tuple(builds.ids), key_category]) # read manually is way faster than using orm res = {} - for (builds_id, key, value) in request.env.cr.fetchall(): - res.setdefault(parents[builds_id], {})[key.split('.', 1)[1]] = value + for (build_id, values) in request.env.cr.fetchall(): + res.setdefault(parents[build_id], {}).update(values) + # we need to update here to manage the post install case: we want to combine stats from all post_install childrens. return res @route(['/runbot/stats//'], type='http', auth="public", website=True, sitemap=False) diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index e38d8e73..20d8738c 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -1035,8 +1035,17 @@ class ConfigStep(models.Model): regex_ids = self.build_stat_regex_ids if not regex_ids: regex_ids = regex_ids.search([('generic', '=', True)]) - key_values = regex_ids._find_in_file(log_path) - self.env['runbot.build.stat']._write_key_values(build, self, key_values) + stats_per_regex = regex_ids._find_in_file(log_path) + if stats_per_regex: + build_stats = [ + { + 'config_step_id': self.id, + 'build_id': build.id, + 'category': category, + 'values': values, + } for category, values in stats_per_regex.items() + ] + self.env['runbot.build.stat'].create(build_stats) except Exception as e: message = '**An error occured while computing statistics of %s:**\n`%s`' % (build.job, str(e).replace('\\n', '\n').replace("\\'", "'")) _logger.exception(message) diff --git a/runbot/models/build_stat.py b/runbot/models/build_stat.py index c74da1b7..c31d76b3 100644 --- a/runbot/models/build_stat.py +++ b/runbot/models/build_stat.py @@ -1,6 +1,7 @@ import logging from odoo import models, fields, api, tools +from ..fields import JsonDictField _logger = logging.getLogger(__name__) @@ -13,7 +14,7 @@ class BuildStat(models.Model): _sql_constraints = [ ( "build_config_key_unique", - "unique (build_id, config_step_id,key)", + "unique (build_id, config_step_id, category)", "Build stats must be unique for the same build step", ) ] @@ -22,89 +23,5 @@ class BuildStat(models.Model): config_step_id = fields.Many2one( "runbot.build.config.step", "Step", ondelete="cascade" ) - key = fields.Char("key", index=True) - value = fields.Float("Value") - - @api.model - def _write_key_values(self, build, config_step, key_values): - if not key_values: - return self - build_stats = [ - { - "config_step_id": config_step.id, - "build_id": build.id, - "key": k, - "value": v, - } - for k, v in key_values.items() - ] - return self.create(build_stats) - - -class RunbotBuildStatSql(models.Model): - - _name = "runbot.build.stat.sql" - _description = "Build stat sql view" - _auto = False - - bundle_id = fields.Many2one("runbot.bundle", string="Bundle", readonly=True) - bundle_name = fields.Char(string="Bundle name", readonly=True) - bundle_sticky = fields.Boolean(string="Sticky", readonly=True) - batch_id = fields.Many2one("runbot.bundle", string="Batch", readonly=True) - trigger_id = fields.Many2one("runbot.trigger", string="Trigger", readonly=True) - trigger_name = fields.Char(string="Trigger name", readonly=True) - - stat_id = fields.Many2one("runbot.build.stat", string="Stat", readonly=True) - key = fields.Char("Key", readonly=True) - value = fields.Float("Value", readonly=True) - - config_step_id = fields.Many2one( - "runbot.build.config.step", string="Config Step", readonly=True - ) - config_step_name = fields.Char(string="Config Step name", readonly=True) - - build_id = fields.Many2one("runbot.build", string="Build", readonly=True) - build_config_id = fields.Many2one("runbot.build.config", string="Config", readonly=True) - build_parent_path = fields.Char('Build Parent path') - build_host = fields.Char(string="Host", readonly=True) - - def init(self): - """ Create SQL view for build stat """ - tools.drop_view_if_exists(self._cr, "runbot_build_stat_sql") - self._cr.execute( - """ CREATE OR REPLACE VIEW runbot_build_stat_sql AS ( - SELECT - (stat.id::bigint*(2^32)+bun.id::bigint) AS id, - stat.id AS stat_id, - stat.key AS key, - stat.value AS value, - step.id AS config_step_id, - step.name AS config_step_name, - bu.id AS build_id, - bp.config_id AS build_config_id, - bu.parent_path AS build_parent_path, - bu.host AS build_host, - bun.id AS bundle_id, - bun.name AS bundle_name, - bun.sticky AS bundle_sticky, - ba.id AS batch_id, - tr.id AS trigger_id, - tr.name AS trigger_name - FROM - runbot_build_stat AS stat - JOIN - runbot_build_config_step step ON stat.config_step_id = step.id - JOIN - runbot_build bu ON bu.id = stat.build_id - JOIN - runbot_build_params bp ON bp.id =bu.params_id - JOIN - runbot_batch_slot bas ON bas.build_id = stat.build_id - JOIN - runbot_trigger tr ON tr.id = bas.trigger_id - JOIN - runbot_batch ba ON ba.id = bas.batch_id - JOIN - runbot_bundle bun ON bun.id = ba.bundle_id - )""" - ) \ No newline at end of file + category = fields.Char("Category", index=True) + values = JsonDictField("Value") diff --git a/runbot/models/build_stat_regex.py b/runbot/models/build_stat_regex.py index 73ff935c..edec91a0 100644 --- a/runbot/models/build_stat_regex.py +++ b/runbot/models/build_stat_regex.py @@ -50,10 +50,11 @@ class BuildStatRegex(models.Model): """ if not os.path.exists(file_path): return {} - key_values = {} + stats_matches = {} with open(file_path, "r") as log_file: data = log_file.read() for build_stat_regex in self: + current_stat_matches = {} for match in re.finditer(build_stat_regex.regex, data): group_dict = match.groupdict() try: @@ -64,10 +65,6 @@ class BuildStatRegex(models.Model): group_dict.get("value"), build_stat_regex.regex ) continue - key = ( - "%s.%s" % (build_stat_regex.name, group_dict["key"]) - if "key" in group_dict - else build_stat_regex.name - ) - key_values[key] = value - return key_values + current_stat_matches[group_dict.get('key', 'value')] = value + stats_matches[build_stat_regex.name] = current_stat_matches + return stats_matches diff --git a/runbot/security/ir.model.access.csv b/runbot/security/ir.model.access.csv index ea3a2446..c2227dc8 100644 --- a/runbot/security/ir.model.access.csv +++ b/runbot/security/ir.model.access.csv @@ -47,13 +47,9 @@ access_runbot_repo_referencetime,runbot_repo_referencetime,runbot.model_runbot_r access_runbot_build_stat_user,runbot_build_stat_user,runbot.model_runbot_build_stat,group_user,1,0,0,0 access_runbot_build_stat_admin,runbot_build_stat_admin,runbot.model_runbot_build_stat,runbot.group_runbot_admin,1,1,1,1 -access_runbot_build_stat_sql_user,runbot_build_stat_sql_user,runbot.model_runbot_build_stat_sql,group_user,1,0,0,0 -access_runbot_build_stat_sql_admin,runbot_build_stat_sql_admin,runbot.model_runbot_build_stat_sql,runbot.group_runbot_admin,1,0,0,0 - access_runbot_build_stat_regex_user,access_runbot_build_stat_regex_user,runbot.model_runbot_build_stat_regex,runbot.group_user,1,0,0,0 access_runbot_build_stat_regex_admin,access_runbot_build_stat_regex_admin,runbot.model_runbot_build_stat_regex,runbot.group_runbot_admin,1,1,1,1 - access_runbot_trigger_user,access_runbot_trigger_user,runbot.model_runbot_trigger,runbot.group_user,1,0,0,0 access_runbot_trigger_runbot_admin,access_runbot_trigger_runbot_admin,runbot.model_runbot_trigger,runbot.group_runbot_admin,1,1,1,1 diff --git a/runbot/tests/test_build_stat.py b/runbot/tests/test_build_stat.py index da661720..8e2872c3 100644 --- a/runbot/tests/test_build_stat.py +++ b/runbot/tests/test_build_stat.py @@ -47,6 +47,7 @@ class TestBuildStatRegex(RunbotCase): def test_build_stat_regex_find_in_file(self): + max_id = self.BuildStat.search([], order="id desc", limit=1).id or 0 file_content = """foo bar 2020-03-02 22:06:58,391 17 INFO xxx odoo.modules.module: odoo.addons.website_blog.tests.test_ui tested in 10.35s, 2501 queries some garbage @@ -58,24 +59,30 @@ nothing to see here ) with patch("builtins.open", mock_open(read_data=file_content)): self.config_step._make_stats(self.build) - - self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_blog.tests.test_ui'), ('value', '=', 2501.0)]), 1) - self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_event.tests.test_ui'), ('value', '=', 2435.0)]), 1) + self.assertEqual( + dict(self.BuildStat.search([('category', '=', 'query_count'), ('id', '>', max_id)]).values), + { + 'website_event.tests.test_ui': 2435.0, + 'website_blog.tests.test_ui': 2501.0 + } + ) # Check unicity with self.assertRaises(IntegrityError): with mute_logger("odoo.sql_db"): with self.cr.savepoint(): # needed to continue tests - self.env["runbot.build.stat"]._write_key_values( - self.build, self.config_step, {'query_count.website_event.tests.test_ui': 2435} - ) + self.env["runbot.build.stat"].create({ + 'build_id': self.build.id, + 'config_step_id': self.config_step.id, + 'category': 'query_count', + 'values': {'website_event.tests.test_ui': 2435}, + } - # minimal test for RunbotBuildStatSql model - # self.assertEqual(self.env['runbot.build.stat.sql'].search_count([('build_id', '=', self.build.id)]), 2) - # TODO FIXME + ) def test_build_stat_regex_generic(self): """ test that regex are not used when generic is False and that _make_stats use all genreic regex if there are no regex on step """ + max_id = self.BuildStat.search([], order="id desc", limit=1).id or 0 file_content = """foo bar odoo.addons.foobar tested in 2s, 25 queries useless 10 @@ -96,12 +103,12 @@ chocolate 15 with patch("builtins.open", mock_open(read_data=file_content)): self.config_step._make_stats(self.build) - self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.foobar'), ('value', '=', 25.0)]), 0) - self.assertEqual(self.BuildStat.search_count([('key', '=', 'useless_count.useless'), ('value', '=', 10.0)]), 0) - self.assertEqual(self.BuildStat.search_count([('key', '=', 'chocolate_count.chocolate'), ('value', '=', 15.0)]), 1) + self.assertEqual(self.BuildStat.search_count([('category', '=', 'query_count'), ('id', '>', max_id)]), 0) + self.assertEqual(self.BuildStat.search_count([('category', '=', 'useless_count'), ('id', '>', max_id)]), 0) + self.assertEqual(dict(self.BuildStat.search([('category', '=', 'chocolate_count'), ('id', '>', max_id)]).values), {'chocolate': 15.0}) def test_build_stat_regex_find_in_file_perf(self): - + max_id = self.BuildStat.search([], order="id desc", limit=1).id or 0 noise_lines = """2020-03-17 13:26:15,472 2376 INFO runbottest odoo.modules.loading: loading runbot/views/build_views.xml 2020-03-10 22:58:34,472 17 INFO 1709329-master-9938b2-all_no_autotag werkzeug: 127.0.0.1 - - [10/Mar/2020 22:58:34] "POST /mail/read_followers HTTP/1.1" 200 - 13 0.004 0.009 2020-03-10 22:58:30,137 17 INFO ? werkzeug: 127.0.0.1 - - [10/Mar/2020 22:58:30] "GET /website/static/src/xml/website.editor.xml HTTP/1.1" 200 - - - - @@ -125,5 +132,10 @@ chocolate 15 with patch("builtins.open", mock_open(read_data=log_data)): self.config_step._make_stats(self.build) - self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_blog.tests.test_ui'), ('value', '=', 2501.0)]), 1) - self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_event.tests.test_ui'), ('value', '=', 2435.0)]), 1) + self.assertEqual( + dict(self.BuildStat.search([('category', '=', 'query_count'), ('id', '>', max_id)]).values), + { + 'website_event.tests.test_ui': 2435.0, + 'website_blog.tests.test_ui': 2501.0 + } + ) diff --git a/runbot/views/build_views.xml b/runbot/views/build_views.xml index ddeb9333..524999bb 100644 --- a/runbot/views/build_views.xml +++ b/runbot/views/build_views.xml @@ -86,17 +86,6 @@ - - - - - - - - - - - diff --git a/runbot/views/menus.xml b/runbot/views/menus.xml index 2d64d6f7..df0bae28 100644 --- a/runbot/views/menus.xml +++ b/runbot/views/menus.xml @@ -9,7 +9,6 @@ - diff --git a/runbot/views/stat_views.xml b/runbot/views/stat_views.xml index 5635296c..09a83d22 100644 --- a/runbot/views/stat_views.xml +++ b/runbot/views/stat_views.xml @@ -1,29 +1,5 @@ - - - runbot.stat.sql.tree - runbot.build.stat.sql - - - - - - - - - - - - - - - - Statistics - runbot.build.stat.sql - tree,graph,pivot - - runbot.build.stat.regex.form runbot.build.stat.regex