diff --git a/runbot/__manifest__.py b/runbot/__manifest__.py index 27e1116c..1a8aefd7 100644 --- a/runbot/__manifest__.py +++ b/runbot/__manifest__.py @@ -27,6 +27,7 @@ 'templates/batch.xml', 'templates/branch.xml', 'templates/build.xml', + 'templates/build_stats.xml', 'templates/bundle.xml', 'templates/commit.xml', 'templates/dashboard.xml', diff --git a/runbot/controllers/frontend.py b/runbot/controllers/frontend.py index f960dee5..4e87b570 100644 --- a/runbot/controllers/frontend.py +++ b/runbot/controllers/frontend.py @@ -7,6 +7,7 @@ import functools import werkzeug.utils import werkzeug.urls +from collections import defaultdict from werkzeug.exceptions import NotFound, Forbidden from odoo.addons.http_routing.models.ir_http import slug @@ -383,3 +384,71 @@ class Runbot(Controller): 'pager': pager } return request.render('runbot.build_error', qctx) + + @route(['/runbot/build/stats/'], type='http', auth="public", website=True) + def build_stats(self, build_id, search=None, **post): + """Build statistics""" + + Build = request.env['runbot.build'] + + build = Build.browse([build_id])[0] + if not build.exists(): + return request.not_found() + + build_stats = defaultdict(dict) + for stat in build.stat_ids.filtered(lambda rec: '.' in rec.key).sorted(key=lambda rec: rec.value, reverse=True): + category, module = stat.key.split('.', maxsplit=1) + value = int(stat.value) if stat.value == int(stat.value) else stat.value + build_stats[category].update({module: value}) + + context = { + 'build': build, + 'build_stats': build_stats, + 'default_category': request.env['ir.model.data'].xmlid_to_res_id('runbot.default_category'), + 'project': build.params_id.trigger_id.project_id, + 'title': 'Build %s statistics' % build.id + } + return request.render("runbot.build_stats", context) + + + @route(['/runbot/stats/'], type='json', auth="public", website=False) + def stats_json(self, bundle_id=False, trigger_id=False, key_category='', max_build_id=False, limit=100, search=None, **post): + """ Json stats """ + trigger_id = trigger_id and int(trigger_id) + bundle_id = bundle_id and int(bundle_id) + max_build_id = max_build_id and int(max_build_id) + limit = int(limit) + limit = min(limit, 1000) + + trigger = request.env['runbot.trigger'].browse(trigger_id) + bundle = request.env['runbot.bundle'].browse(bundle_id) + if not trigger_id or not bundle_id or not trigger.exists() or not bundle.exists(): + return request.not_found() + + builds_domain = [ + ('global_result', '=', 'ok'), ('slot_ids.batch_id.bundle_id', '=', bundle_id), ('params_id.trigger_id', '=', trigger.id), + ] + + if max_build_id: + builds_domain = expression.AND([builds_domain, [('id', '<=', max_build_id)]]) + builds = request.env['runbot.build'].search(builds_domain, order='id desc', limit=limit) + + request.env.cr.execute("SELECT build_id, key, value FROM runbot_build_stat WHERE build_id IN %s AND key like %s", [tuple(builds.ids), '%s.%%' % key_category]) # read manually is way faster than using orm + res = {} + for (builds_id, key, value) in request.env.cr.fetchall(): + res.setdefault(builds_id, {})[key.split('.')[1]] = value + return res + + @route(['/runbot/stats//'], type='http', auth="public", website=True) + def modules_stats(self, bundle, trigger, search=None, **post): + """Modules statistics""" + + categories = request.env['runbot.build.stat.regex'].search([]).mapped('name') + + context = { + 'stats_categories': categories, + 'bundle': bundle, + 'trigger': trigger, + } + + return request.render("runbot.modules_stats", context) diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 33a0ffad..0fd03c22 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -51,10 +51,16 @@ class Trigger(models.Model): hide = fields.Boolean('Hide trigger on main page') manual = fields.Boolean('Only start trigger manually', default=False) - upgrade_dumps_trigger_id = fields.Many2one('runbot.trigger', string= 'Template/complement trigger', tracking=True) + upgrade_dumps_trigger_id = fields.Many2one('runbot.trigger', string='Template/complement trigger', tracking=True) upgrade_step_id = fields.Many2one('runbot.build.config.step', compute="_compute_upgrade_step_id", store=True) ci_url = fields.Char("ci url") ci_description = fields.Char("ci description") + has_stats = fields.Boolean('Has a make_stats config step', compute="_compute_has_stats", store=True) + + @api.depends('config_id.step_order_ids.step_id.make_stats') + def _compute_has_stats(self): + for trigger in self: + trigger.has_stats = any(trigger.config_id.step_order_ids.step_id.mapped('make_stats')) @api.depends('upgrade_dumps_trigger_id', 'config_id', 'config_id.step_order_ids.step_id.job_type') def _compute_upgrade_step_id(self): diff --git a/runbot/static/src/js/stats.js b/runbot/static/src/js/stats.js new file mode 100644 index 00000000..d7042056 --- /dev/null +++ b/runbot/static/src/js/stats.js @@ -0,0 +1,195 @@ + +var config = { + type: 'line', + options: { + legend: { + display: true, + position: 'right', + }, + responsive: true, + tooltips: { + mode: 'point' + }, + scales: { + xAxes: [{ + display: true, + scaleLabel: { + display: true, + labelString: 'Builds' + } + }], + yAxes: [{ + display: true, + scaleLabel: { + display: true, + labelString: 'Queries' + }, + }] + } + } +}; + +config.options.onClick = function(event, activeElements) { + if (activeElements.length === 0){ + var x_label_index = this.scales['x-axis-0'].getValueForPixel(event.x); + var build_id = config.data.labels[x_label_index] + if (event.layerY > this.chartArea.bottom && event.layerY < this.chartArea.bottom + this.scales['x-axis-0'].height){ + config.searchParams['max_build_id'] = build_id; + fetchUpdateChart(); + } + return; + } + window.open('/runbot/build/stats/' + config.data.labels[activeElements[0]._index]); +}; + +function fetch(path, data, then) { + const xhttp = new XMLHttpRequest(); + xhttp.onreadystatechange = function() { + if (this.readyState == 4 && this.status == 200) { + const res = JSON.parse(this.responseText); + then(res.result); + } + }; + xhttp.open("POST", path); + xhttp.setRequestHeader('Content-Type', 'application/json'); + xhttp.send(JSON.stringify({params:data})); + }; + +function random_color(module_name){ + var colors = ['#004acd', '#3658c3', '#4a66ba', '#5974b2', '#6581aa', '#6f8fa3', '#7a9c9d', '#85a899', '#91b596', '#a0c096', '#fdaf56', '#f89a59', '#f1865a', '#e87359', '#dc6158', '#ce5055', '#bf4150', '#ad344b', '#992a45', '#84243d']; + var sum = 0; + for (var i = 0; i < module_name.length; i++) { + sum += module_name.charCodeAt(i); + } + sum = sum % colors.length; + color = colors[sum]; + + return color +}; + + +function process_chart_data(){ + if (Object.keys(config.result).length == 0) + { + config.data = { + labels:[], + datasets: [], + } + return + } + var builds = Object.keys(config.result); + var newer_build_stats = config.result[builds[0]]; + var older_build_stats = config.result[builds.slice(-1)[0]]; + + var mode = document.getElementById('mode_selector').value; + + function display_value(module, build_stats){ + // {'base': 50, 'crm': 25 ...} + if (build_stats === undefined) + build_stats = newer_build_stats; + if (build_stats[module] === undefined) + return NaN; + if (mode == 'normal') + return build_stats[module] + if (older_build_stats[module] === undefined) + return NaN; + return build_stats[module] - older_build_stats[module] + } + + var modules = Object.keys(newer_build_stats); + + modules.sort((m1, m2) => Math.abs(display_value(m2)) - Math.abs(display_value(m1))); + console.log(config.searchParams.nb_dataset) + modules = modules.slice(0, config.searchParams.nb_dataset); + + config.data = { + labels: builds, + datasets: modules.map(function (key){ + return { + label: key, + data: builds.map(build => display_value(key, config.result[build])), + borderColor: random_color(key), + backgroundColor: 'rgba(0, 0, 0, 0)', + lineTension: 0 + } + }) + }; +} + +function fetchUpdateChart() { + var chart_spinner = document.getElementById('chart_spinner'); + chart_spinner.style.visibility = 'visible'; + fetch_params = compute_fetch_params(); + console.log('fetch') + fetch('/runbot/stats/', fetch_params, function(result) { + config.result = result; + chart_spinner.style.visibility = 'hidden'; + updateChart() + }); +}; + +function updateChart(){ + updateUrl(); + process_chart_data(); + if (! window.statsChart) { + var ctx = document.getElementById('canvas').getContext('2d'); + window.statsChart = new Chart(ctx, config); + } else { + window.statsChart.update(); + } +} + +function compute_fetch_params(){ + return { + ...config.searchParams, + bundle_id: document.getElementById('bundle_id').value, + trigger_id: document.getElementById('trigger_id').value, + } +}; + +function updateUrl(){ + window.location.hash = new URLSearchParams(config.searchParams).toString(); +} + +window.onload = function() { + + var mode_selector = document.getElementById('mode_selector'); + var fast_backward_button = document.getElementById('fast_backward_button'); + + config.searchParams = { + limit: 25, + max_build_id: 0, + key_category: 'module_loading_queries', + mode: 'normal', + nb_dataset: 20, + }; + localParams = ['mode', 'nb_dataset'] + + for([key, value] of new URLSearchParams(window.location.hash.replace("#","?"))){ + config.searchParams[key] = value; + } + + + for([key, value] of Object.entries(config.searchParams)){ + var selector = document.getElementById(key + '_selector'); + if (selector != null){ + selector.value = value; + selector.onchange = function(){ + var id = this.id.replace('_selector', ''); + config.searchParams[this.id.replace('_selector', '')] = this.value; + if (localParams.indexOf(id) == -1){ + fetchUpdateChart(); + } else { + updateChart() + } + } + } + } + + fast_backward_button.onclick = function(){ + config.searchParams['max_build_id'] = Object.keys(config.result)[0]; + fetchUpdateChart(); + } + + fetchUpdateChart(); +}; diff --git a/runbot/templates/assets.xml b/runbot/templates/assets.xml index 92a2d16c..3aee9a2d 100644 --- a/runbot/templates/assets.xml +++ b/runbot/templates/assets.xml @@ -4,6 +4,7 @@ diff --git a/runbot/templates/build.xml b/runbot/templates/build.xml index 4ec69b6e..0ba3ee32 100644 --- a/runbot/templates/build.xml +++ b/runbot/templates/build.xml @@ -138,9 +138,11 @@ Total time:
- Trigger: - -
+ + Stats: + Build +
+

diff --git a/runbot/templates/build_stats.xml b/runbot/templates/build_stats.xml new file mode 100644 index 00000000..0360a2cd --- /dev/null +++ b/runbot/templates/build_stats.xml @@ -0,0 +1,111 @@ + + + + + + + + diff --git a/runbot/templates/bundle.xml b/runbot/templates/bundle.xml index 7fc9eb85..d366aed7 100644 --- a/runbot/templates/bundle.xml +++ b/runbot/templates/bundle.xml @@ -21,6 +21,7 @@ + diff --git a/runbot/templates/utils.xml b/runbot/templates/utils.xml index d6290c1d..ce513c8d 100644 --- a/runbot/templates/utils.xml +++ b/runbot/templates/utils.xml @@ -306,5 +306,20 @@ + + + diff --git a/runbot/views/repo_views.xml b/runbot/views/repo_views.xml index 98eba586..2033f873 100644 --- a/runbot/views/repo_views.xml +++ b/runbot/views/repo_views.xml @@ -26,6 +26,7 @@ +