From 3ed0ef658a625f5098cd245014ad21689ddd3046 Mon Sep 17 00:00:00 2001 From: William Braeckman Date: Mon, 13 Jan 2025 11:49:58 +0100 Subject: [PATCH] [IMP] runbot: display stats per bundle When selecting multiple bundles, differentiate stats between bundles instead of displaying them all as the same. --- runbot/controllers/frontend.py | 24 ++- runbot/models/build_stat.py | 6 +- runbot/static/src/stats/cache.js | 29 ++++ runbot/static/src/stats/stats_chart.js | 208 +++++++++++++++--------- runbot/static/src/stats/stats_chart.xml | 12 +- runbot/static/src/stats/stats_config.js | 21 +-- runbot/static/src/stats/stats_root.js | 2 + runbot/templates/build_stats.xml | 2 + 8 files changed, 195 insertions(+), 109 deletions(-) create mode 100644 runbot/static/src/stats/cache.js diff --git a/runbot/controllers/frontend.py b/runbot/controllers/frontend.py index 25ca1f62..97678e04 100644 --- a/runbot/controllers/frontend.py +++ b/runbot/controllers/frontend.py @@ -3,7 +3,6 @@ import datetime import werkzeug import logging import functools -import itertools import werkzeug.utils import werkzeug.urls @@ -600,7 +599,7 @@ class Runbot(Controller): builds = request.env['runbot.build'].with_context(active_test=False) if center_build_id: builds = builds.search( - expression.AND([builds_domain, [('id', '>=', center_build_id)]]), + expression.AND([builds_domain, [('id', '>', center_build_id)]]), order='id', limit=limit/2) builds_domain = expression.AND([builds_domain, [('id', '<=', center_build_id)]]) limit -= len(builds) @@ -612,13 +611,28 @@ class Runbot(Controller): builds = builds.search([('id', 'child_of', builds.ids)]) parents = {b.id: b.top_parent.id for b in builds.with_context(prefetch_fields=False)} + # Prefetch bundle name, we need to be able to bind a + builds.with_context(prefetch_fields=False).fetch([ + 'create_date', 'slot_ids', + ]) + res_arr = [ + { + 'id': b.id, + 'values': {}, + 'create_date': b.create_date.isoformat(), + 'bundle_id': b.slot_ids.batch_id.bundle_id.id, + } + for b in builds.with_context(prefetch_fields=False).sorted('id') + ] + res_dict = {r['id']: r['values'] for r in res_arr} request.env.cr.execute("SELECT build_id, values FROM runbot_build_stat WHERE build_id IN %s AND category = %s", [tuple(builds.ids), key_category]) # read manually is way faster than using orm - res = {} for (build_id, values) in request.env.cr.fetchall(): if values: - res.setdefault(parents[build_id], {}).update(values) + res_dict[parents[build_id]].update(values) # we need to update here to manage the post install case: we want to combine stats from all post_install childrens. - return res + # Filter out results without values + res_arr = [r for r in res_arr if r['values']] + return res_arr @route(['/runbot/stats//'], type='http', auth="public", website=True, sitemap=False) def modules_stats(self, bundle, trigger, search=None, **post): diff --git a/runbot/models/build_stat.py b/runbot/models/build_stat.py index c31d76b3..5a74629a 100644 --- a/runbot/models/build_stat.py +++ b/runbot/models/build_stat.py @@ -13,8 +13,8 @@ class BuildStat(models.Model): _sql_constraints = [ ( - "build_config_key_unique", - "unique (build_id, config_step_id, category)", + "category_build_config_step_unique_key", + "unique (category, build_id, config_step_id)", "Build stats must be unique for the same build step", ) ] @@ -23,5 +23,5 @@ class BuildStat(models.Model): config_step_id = fields.Many2one( "runbot.build.config.step", "Step", ondelete="cascade" ) - category = fields.Char("Category", index=True) + category = fields.Char("Category") values = JsonDictField("Value") diff --git a/runbot/static/src/stats/cache.js b/runbot/static/src/stats/cache.js new file mode 100644 index 00000000..cb33c8a4 --- /dev/null +++ b/runbot/static/src/stats/cache.js @@ -0,0 +1,29 @@ +/** @odoo-module **/ + + +/** + * @typedef Bundle + * + * @property {Number} id + * @property {String} name + */ + +const bundleNameCache = {}; // id to name + +/** + * Returns the name of the bundle according to the cache. + * + * @param {Number} bundleId id of the bundle + */ +export const getBundleName = (bundleId) => { + return bundleNameCache[bundleId] || bundleId.toString(); +} + +/** + * Populates the cache when a new list of bundle is loaded. + * + * @param {Bundle[]} bundles list of bundles + */ +export const populateCache = (bundles) => { + bundles.forEach(({ id, name }) => bundleNameCache[id] = name); +} diff --git a/runbot/static/src/stats/stats_chart.js b/runbot/static/src/stats/stats_chart.js index 7b37553d..82173389 100644 --- a/runbot/static/src/stats/stats_chart.js +++ b/runbot/static/src/stats/stats_chart.js @@ -5,8 +5,17 @@ import { Component, useEffect, useRef, useState } from '@odoo/owl'; import { debounce, filterKeys, randomColor } from '@runbot/utils'; import { useConfig, onConfigChange } from '@runbot/stats/use_config'; import { Chart } from '@runbot/chartjs'; +import { getBundleName } from '@runbot/stats/cache'; +/** + * @typedef StatsQueryResult + * + * @property {Number} id id of the build + * @property {Object.} values object with value name as key and value as value + * @property {String} create_date creation date of the build + * @property {Number} bundle_id bundle id of the build + */ export class StatsChart extends Component { static template = 'runbot.StatsChart'; static props = { @@ -39,6 +48,7 @@ export class StatsChart extends Component { }, scales: { x: { + type: 'time', display: true, scaleLabel: { display: true, @@ -58,11 +68,16 @@ export class StatsChart extends Component { if (activeElements.length === 0) { return; } - const build_id = this.chartConfig.data.labels[activeElements[0].index]; + const { datasetIndex, index } = activeElements[0]; + const queryResult = this.chartConfig.data.datasets[datasetIndex].data[index]._queryResult; + if (!queryResult) { + return console.error('Queryresult not present for datasetIndex and index ', datasetIndex, index); + } + const { id } = queryResult; if (shiftKey) { - this.config.center_build_id = build_id; + this.config.center_build_id = id; } else { - window.open(`/runbot/build/stats/${build_id}`); + window.open(`/runbot/build/stats/${id}`); } } }, @@ -142,89 +157,132 @@ export class StatsChart extends Component { const { display_aggregate: aggregate, mode, + nb_dataset, } = this.config; - const { data } = this.state; - const builds = Object.keys(data); - const newestBuildStats = data[builds[builds.length - 1]]; - const oldestBuildStats = data[builds[0]]; - const keys = Object.keys(newestBuildStats); - let idx = keys.indexOf('Aggregate Sum'); - if (aggregate === 'sum' && idx === -1) { - keys.push('Aggregate Sum'); - Object.values(data).forEach((buildData) => { - buildData['Aggregate Sum'] = Object.values(buildData).reduce((a, b) => a + b, 0); - }); - } else if (aggregate !== 'sum' && idx !== -1) { - keys.splice(idx, 1); + + /** @type {StatsQueryResult[]} */ + const data = this.state.data; + const bundles = new Set(data.map(({bundle_id}) => bundle_id)); + const hasMultiBundle = bundles.size > 1; + /** + * Gets the label (group by key) for the given queryResult + * + * @param {StatsQueryResult} queryResult + * @param {String} valueName + */ + const getLabel = (queryResult, valueName) => { + // Q: Change to bundleName if not main bundle? + if (hasMultiBundle) { + return `${valueName} (${getBundleName(queryResult.bundle_id)})`; + } + return `${valueName}`; } - idx = keys.indexOf('Aggregate Average'); - if (aggregate === 'average' && idx === -1) { - keys.push('Aggregate Average'); - Object.values(data).forEach((buildData) => { - buildData['Aggregate Average'] = (Object.values(buildData).reduce((a, b) => a + b, 0) / Object.values(buildData).length); - }); - } else if (aggregate !== 'average' && idx !== -1) { - keys.splice(idx, 1); - } - // Mapping of keys to their sort value - const sortValues = keys.reduce( - (dict, key) => { - const getValue = () => { - if (mode === 'normal') { - return newestBuildStats[key]; - } else if (mode === 'alpha') { - return key; - } else if (mode === 'change_count') { - return builds.reduce((agg, build, buildIdx) => { - const currentBuild = data[build]; - const current = currentBuild[key]; - const previous = buildIdx === 0 ? undefined : data[builds[buildIdx - 1]][key]; - if (previous !== undefined && current !== undefined && previous != current) { - agg += 1; - } - return agg; - }, 0); - } else if (mode === 'difference') { - return Math.abs( - newestBuildStats[key] - (oldestBuildStats[key] || 0) - ); + // Group values by (valueName, bundle Id) we want to separate bundle ids. + const datasets = Object.values(data.reduce((agg, queryResult) => { + Object.entries(queryResult.values).forEach(([valueName, value]) => { + const label = getLabel(queryResult, valueName); + if (!(agg[label])) { + agg[label] = { + label, + data: [], + borderColor: randomColor(label), + backgroundColor: 'rgba(0, 0, 0, 0)', + lineTension: 0, + hidden: false, + _queryResult: queryResult, } } - dict[key] = getValue(); - return dict; - }, {}, - ); - keys.sort((k1, k2) => { - if (mode === 'alpha') { - return sortValues[k1].localeCompare(sortValues[k2]); + agg[label].data.push({ + x: queryResult.create_date, + y: value, + _queryResult: queryResult, + }); + }) + return agg; + }, {})); + + // Compute selected aggregate + if (aggregate != 'none') { + const queryResultsByBundle = data.reduce((agg, queryResult) => { + if (!(agg[queryResult.bundle_id])) { + agg[queryResult.bundle_id] = []; + } + agg[[queryResult.bundle_id]].push(queryResult); + return agg + }, {}); + Object.values(queryResultsByBundle).forEach((queryResults) => { + const newData = queryResults.map((qs) => { + return { + x: qs.create_date, + y: Object.values(qs.values).reduce((s, v) => s + v, 0), + _queryResult: qs, + } + }); + let label = getLabel(datasets[0]._queryResult, 'Aggregate Sum'); + if (aggregate === 'average') { + label = getLabel(datasets[0]._queryResult, 'Aggregate Average'); + newData.forEach(d => d.y /= Object.values(d._queryResult.values).length); + } + datasets.push({ + label, + data: newData, + borderColor: randomColor(label), + backgroundColor: 'rgba(0, 0, 0, 0)', + lineTension: 0, + hidden: false, + _queryResult: datasets[0]._queryResult, + }); + }); + } + // Compute a sorting value for each dataset, sort + // Also recompute data if mode requires it. + datasets.forEach((dataset) => { + const getSortValue = () => { + if (mode === 'normal') { + return dataset.data[0].y; + } else if (mode === 'alpha') { + return dataset.label; + } else if (mode === 'change_count') { + return dataset.data.reduce((agg, {y}, dataIdx) => { + const previous = dataIdx === 0 ? undefined : dataset.data[dataIdx - 1].y; + if (previous !== undefined && y !== undefined && previous != y) { + agg += 1; + } + return agg; + }, 0); + } else if (mode === 'difference') { + return Math.abs( + dataset.data[dataset.data.length - 1].y - dataset.data[0].y + ); + } + } + dataset._sortValue = getSortValue(); + if (mode === 'change_count' || mode === 'difference') { + const firstValue = dataset.data[0].y; + dataset.data = dataset.data.map(d => { + return { + ...d, + y: d.y - firstValue, + }; + }); } - return sortValues[k2] - sortValues[k1] }); + datasets.sort((ds1, ds2) => { + if (mode === 'alpha') { + return ds1._sortValue.localeCompare(ds2._sortValue); + } + return ds2._sortValue - ds1._sortValue; + }); + // Change visibility of datasets according to config let visibleKeys; - if (this.config.nb_dataset !== -1) { - visibleKeys = new Set(keys.slice(0, this.config.nb_dataset)); + if (nb_dataset !== -1) { + visibleKeys = new Set(datasets.slice(0, nb_dataset).map(ds => ds.label)); } else { visibleKeys = new Set(this.config.getVisibleKeys()); } - const getDisplayValue = (key, build) => { - if (build[key] === undefined) { - return NaN; - } - if (mode === 'normal' || mode === 'alpha') { - return build[key]; - } - return build[key] - (oldestBuildStats[key] || 0) - } + datasets.forEach(ds => ds.hidden = !visibleKeys.has(ds.label)); this.chartConfig.data = { - labels: builds, - datasets: keys.map((key) => ({ - label: key, - data: builds.map(build => getDisplayValue(key, data[build])), - borderColor: randomColor(key), - backgroundColor: 'rgba(0, 0, 0, 0)', - lineTension: 0, - hidden: !visibleKeys.has(key), - })), + datasets, }; } diff --git a/runbot/static/src/stats/stats_chart.xml b/runbot/static/src/stats/stats_chart.xml index cba6c03a..47556b2d 100644 --- a/runbot/static/src/stats/stats_chart.xml +++ b/runbot/static/src/stats/stats_chart.xml @@ -16,11 +16,11 @@
@@ -62,7 +62,7 @@
-
+
  • { - return bundleNameCache[bundleId] || bundleId.toString(); -} - -/** - * Populates the cache when a new list of bundle is loaded. - * - * @param {Bundle[]} bundles list of bundles - */ -const populateCache = (bundles) => { - bundles.forEach(({ id, name }) => bundleNameCache[id] = name); -} - export class StatsConfig extends Component { static template = 'runbot.StatsConfig'; static props = { diff --git a/runbot/static/src/stats/stats_root.js b/runbot/static/src/stats/stats_root.js index 7698a2d0..76b27ed7 100644 --- a/runbot/static/src/stats/stats_root.js +++ b/runbot/static/src/stats/stats_root.js @@ -7,6 +7,7 @@ import { StatsConfig } from '@runbot/stats/stats_config'; import { StatsChart } from '@runbot/stats/stats_chart'; import { useConfig } from '@runbot/stats/use_config'; import { UrlUpdater } from '@runbot/stats/url_updater'; +import { populateCache } from '@runbot/stats/cache'; export class StatsRoot extends Component { @@ -54,6 +55,7 @@ export class StatsRoot extends Component { setup() { // Initialize shared configuration for children components. useConfig(false); + populateCache([this.props.bundle]); } } diff --git a/runbot/templates/build_stats.xml b/runbot/templates/build_stats.xml index 71bde452..31972423 100644 --- a/runbot/templates/build_stats.xml +++ b/runbot/templates/build_stats.xml @@ -77,6 +77,8 @@
    This page requires javascript to load
    +