NextERP/runbot
5
0
Fork 0
mirror of https://github.com/odoo/runbot.git synced 2025-03-15 23:45:44 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

[IMP] runbot: add a chart page for build stats

Browse Source 
Since 360e31ade4, it's possible to add statistics values to build
results but there was no practical way to analyze them.

With this commit, there is a new button on the bundle page that leads to
a chart page that displays those values.

The default reference build is last known good build of the bundle.
Values are filtered by key and only the most significant values are
displayed. The user can then refine the chart by changing the reference
build or the key and a few other options.

Co-author: Xavier-Do <xdo@odoo.com>
This commit is contained in:
Christophe Monniez 2021-03-16 10:45:47 +01:00 committed by xdo
parent 19c312d92c
commit 57bd00672d
10 changed files with 406 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
runbot/__manifest__.py
View File

@ -27,6 +27,7 @@
'templates/batch.xml',
'templates/branch.xml',
'templates/build.xml',
'templates/build_stats.xml',
'templates/bundle.xml',
'templates/commit.xml',
'templates/dashboard.xml',

69
runbot/controllers/frontend.py
View File

@ -7,6 +7,7 @@ import functools
import werkzeug.utils
import werkzeug.urls
from collections import defaultdict
from werkzeug.exceptions import NotFound, Forbidden
from odoo.addons.http_routing.models.ir_http import slug
@ -383,3 +384,71 @@ class Runbot(Controller):
'pager': pager
}
return request.render('runbot.build_error', qctx)
@route(['/runbot/build/stats/<int:build_id>'], type='http', auth="public", website=True)
def build_stats(self, build_id, search=None, **post):
"""Build statistics"""
Build = request.env['runbot.build']
build = Build.browse([build_id])[0]
if not build.exists():
return request.not_found()
build_stats = defaultdict(dict)
for stat in build.stat_ids.filtered(lambda rec: '.' in rec.key).sorted(key=lambda rec: rec.value, reverse=True):
category, module = stat.key.split('.', maxsplit=1)
value = int(stat.value) if stat.value == int(stat.value) else stat.value
build_stats[category].update({module: value})
context = {
'build': build,
'build_stats': build_stats,
'default_category': request.env['ir.model.data'].xmlid_to_res_id('runbot.default_category'),
'project': build.params_id.trigger_id.project_id,
'title': 'Build %s statistics' % build.id
}
return request.render("runbot.build_stats", context)
@route(['/runbot/stats/'], type='json', auth="public", website=False)
def stats_json(self, bundle_id=False, trigger_id=False, key_category='', max_build_id=False, limit=100, search=None, **post):
""" Json stats """
trigger_id = trigger_id and int(trigger_id)
bundle_id = bundle_id and int(bundle_id)
max_build_id = max_build_id and int(max_build_id)
limit = int(limit)
limit = min(limit, 1000)
trigger = request.env['runbot.trigger'].browse(trigger_id)
bundle = request.env['runbot.bundle'].browse(bundle_id)
if not trigger_id or not bundle_id or not trigger.exists() or not bundle.exists():
return request.not_found()
builds_domain = [
('global_result', '=', 'ok'), ('slot_ids.batch_id.bundle_id', '=', bundle_id), ('params_id.trigger_id', '=', trigger.id),
]
if max_build_id:
builds_domain = expression.AND([builds_domain, [('id', '<=', max_build_id)]])
builds = request.env['runbot.build'].search(builds_domain, order='id desc', limit=limit)
request.env.cr.execute("SELECT build_id, key, value FROM runbot_build_stat WHERE build_id IN %s AND key like %s", [tuple(builds.ids), '%s.%%' % key_category]) # read manually is way faster than using orm
res = {}
for (builds_id, key, value) in request.env.cr.fetchall():
res.setdefault(builds_id, {})[key.split('.')[1]] = value
return res
@route(['/runbot/stats/<model("runbot.bundle"):bundle>/<model("runbot.trigger"):trigger>'], type='http', auth="public", website=True)
def modules_stats(self, bundle, trigger, search=None, **post):
"""Modules statistics"""
categories = request.env['runbot.build.stat.regex'].search([]).mapped('name')
context = {
'stats_categories': categories,
'bundle': bundle,
'trigger': trigger,
}
return request.render("runbot.modules_stats", context)

8
runbot/models/repo.py
View File

@ -51,10 +51,16 @@ class Trigger(models.Model):
hide = fields.Boolean('Hide trigger on main page')
manual = fields.Boolean('Only start trigger manually', default=False)
upgrade_dumps_trigger_id = fields.Many2one('runbot.trigger', string= 'Template/complement trigger', tracking=True)
upgrade_dumps_trigger_id = fields.Many2one('runbot.trigger', string='Template/complement trigger', tracking=True)
upgrade_step_id = fields.Many2one('runbot.build.config.step', compute="_compute_upgrade_step_id", store=True)
ci_url = fields.Char("ci url")
ci_description = fields.Char("ci description")
has_stats = fields.Boolean('Has a make_stats config step', compute="_compute_has_stats", store=True)
@api.depends('config_id.step_order_ids.step_id.make_stats')
def _compute_has_stats(self):
for trigger in self:
trigger.has_stats = any(trigger.config_id.step_order_ids.step_id.mapped('make_stats'))
@api.depends('upgrade_dumps_trigger_id', 'config_id', 'config_id.step_order_ids.step_id.job_type')
def _compute_upgrade_step_id(self):

195
runbot/static/src/js/stats.js Normal file
View File

@ -0,0 +1,195 @@
var config = {
type: 'line',
options: {
legend: {
display: true,
position: 'right',
},
responsive: true,
tooltips: {
mode: 'point'
},
scales: {
xAxes: [{
display: true,
scaleLabel: {
display: true,
labelString: 'Builds'
}
}],
yAxes: [{
display: true,
scaleLabel: {
display: true,
labelString: 'Queries'
},
}]
}
}
};
config.options.onClick = function(event, activeElements) {
if (activeElements.length === 0){
var x_label_index = this.scales['x-axis-0'].getValueForPixel(event.x);
var build_id = config.data.labels[x_label_index]
if (event.layerY > this.chartArea.bottom && event.layerY < this.chartArea.bottom + this.scales['x-axis-0'].height){
config.searchParams['max_build_id'] = build_id;
fetchUpdateChart();
}
return;
}
window.open('/runbot/build/stats/' + config.data.labels[activeElements[0]._index]);
};
function fetch(path, data, then) {
const xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) {
const res = JSON.parse(this.responseText);
then(res.result);
}
};
xhttp.open("POST", path);
xhttp.setRequestHeader('Content-Type', 'application/json');
xhttp.send(JSON.stringify({params:data}));
};
function random_color(module_name){
var colors = ['#004acd', '#3658c3', '#4a66ba', '#5974b2', '#6581aa', '#6f8fa3', '#7a9c9d', '#85a899', '#91b596', '#a0c096', '#fdaf56', '#f89a59', '#f1865a', '#e87359', '#dc6158', '#ce5055', '#bf4150', '#ad344b', '#992a45', '#84243d'];
var sum = 0;
for (var i = 0; i < module_name.length; i++) {
sum += module_name.charCodeAt(i);
}
sum = sum % colors.length;
color = colors[sum];
return color
};
function process_chart_data(){
if (Object.keys(config.result).length == 0)
{
config.data = {
labels:[],
datasets: [],
}
return
}
var builds = Object.keys(config.result);
var newer_build_stats = config.result[builds[0]];
var older_build_stats = config.result[builds.slice(-1)[0]];
var mode = document.getElementById('mode_selector').value;
function display_value(module, build_stats){
// {'base': 50, 'crm': 25 ...}
if (build_stats === undefined)
build_stats = newer_build_stats;
if (build_stats[module] === undefined)
return NaN;
if (mode == 'normal')
return build_stats[module]
if (older_build_stats[module] === undefined)
return NaN;
return build_stats[module] - older_build_stats[module]
}
var modules = Object.keys(newer_build_stats);
modules.sort((m1, m2) => Math.abs(display_value(m2)) - Math.abs(display_value(m1)));
console.log(config.searchParams.nb_dataset)
modules = modules.slice(0, config.searchParams.nb_dataset);
config.data = {
labels: builds,
datasets: modules.map(function (key){
return {
label: key,
data: builds.map(build => display_value(key, config.result[build])),
borderColor: random_color(key),
backgroundColor: 'rgba(0, 0, 0, 0)',
lineTension: 0
}
})
};
}
function fetchUpdateChart() {
var chart_spinner = document.getElementById('chart_spinner');
chart_spinner.style.visibility = 'visible';
fetch_params = compute_fetch_params();
console.log('fetch')
fetch('/runbot/stats/', fetch_params, function(result) {
config.result = result;
chart_spinner.style.visibility = 'hidden';
updateChart()
});
};
function updateChart(){
updateUrl();
process_chart_data();
if (! window.statsChart) {
var ctx = document.getElementById('canvas').getContext('2d');
window.statsChart = new Chart(ctx, config);
} else {
window.statsChart.update();
}
}
function compute_fetch_params(){
return {
...config.searchParams,
bundle_id: document.getElementById('bundle_id').value,
trigger_id: document.getElementById('trigger_id').value,
}
};
function updateUrl(){
window.location.hash = new URLSearchParams(config.searchParams).toString();
}
window.onload = function() {
var mode_selector = document.getElementById('mode_selector');
var fast_backward_button = document.getElementById('fast_backward_button');
config.searchParams = {
limit: 25,
max_build_id: 0,
key_category: 'module_loading_queries',
mode: 'normal',
nb_dataset: 20,
};
localParams = ['mode', 'nb_dataset']
for([key, value] of new URLSearchParams(window.location.hash.replace("#","?"))){
config.searchParams[key] = value;
}
for([key, value] of Object.entries(config.searchParams)){
var selector = document.getElementById(key + '_selector');
if (selector != null){
selector.value = value;
selector.onchange = function(){
var id = this.id.replace('_selector', '');
config.searchParams[this.id.replace('_selector', '')] = this.value;
if (localParams.indexOf(id) == -1){
fetchUpdateChart();
} else {
updateChart()
}
}
}
}
fast_backward_button.onclick = function(){
config.searchParams['max_build_id'] = Object.keys(config.result)[0];
fetchUpdateChart();
}
fetchUpdateChart();
};

1
runbot/templates/assets.xml
View File

@ -4,6 +4,7 @@
<xpath expr="." position="inside">
<link rel="stylesheet" href="/runbot/static/src/css/runbot.scss"/>
<script type="text/javascript" src="/runbot/static/src/js/runbot.js"/>
<script type="text/javascript" src="/web/static/lib/Chart/Chart.js"></script>
</xpath>
</template>
</data>

8
runbot/templates/build.xml
View File

@ -138,9 +138,11 @@
<b>Total time:</b>
<t t-esc="build.get_formated_build_time()"/>
<br/>
<b>Trigger:</b>
<t t-esc="build.params_id.trigger_id.name"/>
<br/>
<t t-if="build.stat_ids">
<b>Stats:</b>
<a t-attf-href="/runbot/build/stats/{{build.id}}">Build <t t-esc="build.id"/></a>
<br/>
</t>
<br/>
</td>
<td t-if="build.children_ids">

111
runbot/templates/build_stats.xml Normal file
View File

@ -0,0 +1,111 @@
<?xml version="1.0" encoding="utf-8"?>
<odoo>
<data>
<template id="runbot.build_stats">
<t t-call='website.layout'>
<t t-set="bundles" t-value="build.slot_ids.mapped('batch_id.bundle_id')"/>
<div>
<div class="row">
<div class="col-md-4">
<div class="bg-success-light">
<b>Build: </b><a t-attf-href="/runbot/build/{{build.id}}"><t t-esc="build.id"/></a><br/>
<t t-if="build.description">
<b>Description:</b>
<t t-raw="build.md_description"/>
<br/>
</t>
<b>Date: </b><t t-esc="build.create_date" /><br/>
<b>Config: </b><t t-esc="build.params_id.config_id.name" /><br/>
<b>Bundle(s): </b>
<t t-foreach="bundles" t-as="bundle">
<a t-attf-href="/runbot/bundle/{{bundle.id}}"><t t-esc="bundle.name" /></a>
</t><br/>
<t t-foreach="build.params_id.sudo().commit_link_ids" t-as="build_commit">
<b>Commit:</b>
<a t-attf-href="/runbot/commit/{{build_commit.commit_id.id}}">
<t t-esc="build_commit.commit_id.dname"/>
</a>
<a t-att-href="'https://%s/commit/%s' % (build_commit.branch_id.remote_id.base_url, build_commit.commit_id.name)" class="btn btn-sm text-left" title="View Commit on Github"><i class="fa fa-github"/></a>
<t t-if="build_commit.match_type in ('default', 'pr_target', 'prefix') ">
from base branch
<br/>
</t>
<div t-else="" class="ml-3">
<b>Subject:</b>
<t t-esc="build_commit.commit_id.subject"/>
<br/>
<b>Author:</b>
<t t-esc="build_commit.commit_id.author"/>
<br/>
<b>Committer:</b>
<t t-esc="build_commit.commit_id.committer"/>
<br/>
</div>
</t>
<b>Version:</b>
<t t-esc="build.params_id.version_id.name"/>
<br/>
</div>
</div>
<div t-foreach="sorted(build_stats.keys())" t-as="category" class="col-md-4">
<h3><t t-esc="category.title().replace('_', ' ')"/></h3>
<table class="table table-condensed table-responsive table-stripped">
<tr t-foreach="build_stats[category].keys()" t-as="module">
<td><t t-esc="module"/></td>
<td><t t-esc="build_stats[category][module]"/></td>
</tr>
</table>
</div>
<div t-if="not build_stats" class="col-md-12 alert alert-warning">No stats records found for this build</div>
</div>
</div>
</t>
</template>
<template id="runbot.modules_stats">
<t t-call='website.layout'>
<input type="hidden" id="bundle_id" t-att-value="bundle.id"/>
<input type="hidden" id="trigger_id" t-att-value="trigger.id"/>
<div class="container-fluid">
<nav class="navbar navbar-light">
<div class="container">
<b>Bundle:</b><t t-esc="bundle.name"/><br/>
<b>Trigger:</b><t t-esc="trigger.name"/>
<b>Stat Category:</b>
<select id="key_category_selector" class="form-select" aria-label="Stat Category">
<option t-foreach="stats_categories" t-as="category" t-attf-value="{{category}}"><t t-esc="category.replace('_',' ').title()"/></option>
</select>
<b>Mode: </b>
<select id="mode_selector" class="form-select" aria-label="Display mode">
<option title="Real Values" selected="selected" value="normal">Normal</option>
<option title="Delta With Reference Build Values" value="difference">Difference</option>
</select>
<b>Nb of builds:</b>
<select id="limit_selector" class="form-select" aria-label="Number Of Builds">
<option value="10">10</option>
<option value="25">25</option>
<option value="50">50</option>
<option selected="selected" value="100">100</option>
<option value="150">250</option>
</select>
<b>Nb datasets:</b>
<select id="nb_dataset_selector" class="form-select" aria-label="Number Of Builds">
<option value="10">10</option>
<option value="20">20</option>
<option value="50">50</option>
<option selected="selected" value="100">100</option>
</select>
<button id="fast_backward_button" class="btn btn-default" title="Previous Builds" aria-label="Previous Builds">
<i t-attf-class="fa fa-fast-backward"/>
</button>
<i id="chart_spinner" class="fa fa-2x fa-circle-o-notch fa-spin"/>
</div>
</nav>
<canvas id="canvas"></canvas>
</div>
</t>
<script type="text/javascript" src="/runbot/static/src/js/stats.js"></script>
</template>
</data>
</odoo>

1
runbot/templates/bundle.xml
View File

@ -21,6 +21,7 @@
<i class="fa fa-fast-forward"/>
</a>
<t t-call="runbot.branch_copy_button"/>
<t t-call="runbot.bundle_stats_dropdown"/>
</div>
</span>
<span class="pull-right">

15
runbot/templates/utils.xml
View File

@ -306,5 +306,20 @@
<i t-attf-class="fa fa-clipboard"/>
</button>
</template>
<template id="runbot.bundle_stats_dropdown">
<button t-attf-class="btn btn-default dropdown-toggle" data-toggle="dropdown" title="Bundle Stats" aria-label="Bundle Stats" aria-expanded="false">
<i t-attf-class="fa fa-bar-chart"/>
<span class="caret"/>
</button>
<div class="dropdown-menu dropdown-menu-right" role="menu">
<t t-foreach="project.trigger_ids" t-as="trigger">
<a class="dropdown-item" t-if="trigger.has_stats" t-attf-href="/runbot/stats/{{bundle.id}}/{{trigger.id}}">
<t t-esc="trigger.name" />
</a>
</t>
</div>
</template>
</data>
</odoo>

1
runbot/views/repo_views.xml
View File

@ -26,6 +26,7 @@
<field name="ci_context"/>
<field name="ci_url"/>
<field name="ci_description"/>
<field name="has_stats"/>
</group>
</sheet>
</form>