[IMP] runbot: display stats per bundle

When selecting multiple bundles, differentiate stats between bundles
instead of displaying them all as the same.

runbot/controllers/frontend.py

@@ -3,7 +3,6 @@ import datetime
 import werkzeug
 import logging
 import functools
-import itertools
 
 import werkzeug.utils
 import werkzeug.urls
@@ -600,7 +599,7 @@ class Runbot(Controller):
         builds = request.env['runbot.build'].with_context(active_test=False)
         if center_build_id:
             builds = builds.search(
-                expression.AND([builds_domain, [('id', '>=', center_build_id)]]),
+                expression.AND([builds_domain, [('id', '>', center_build_id)]]),
                 order='id', limit=limit/2)
             builds_domain = expression.AND([builds_domain, [('id', '<=', center_build_id)]])
             limit -= len(builds)
@@ -612,13 +611,28 @@ class Runbot(Controller):
         builds = builds.search([('id', 'child_of', builds.ids)])
 
         parents = {b.id: b.top_parent.id for b in builds.with_context(prefetch_fields=False)}
+        # Prefetch bundle name, we need to be able to bind a
+        builds.with_context(prefetch_fields=False).fetch([
+            'create_date', 'slot_ids',
+        ])
+        res_arr = [
+            {
+                'id': b.id,
+                'values': {},
+                'create_date': b.create_date.isoformat(),
+                'bundle_id': b.slot_ids.batch_id.bundle_id.id,
+            }
+            for b in builds.with_context(prefetch_fields=False).sorted('id')
+        ]
+        res_dict = {r['id']: r['values'] for r in res_arr}
         request.env.cr.execute("SELECT build_id, values FROM runbot_build_stat WHERE build_id IN %s AND category = %s", [tuple(builds.ids), key_category])  # read manually is way faster than using orm
-        res = {}
         for (build_id, values) in request.env.cr.fetchall():
             if values:
-                res.setdefault(parents[build_id], {}).update(values)
+                res_dict[parents[build_id]].update(values)
             # we need to update here to manage the post install case: we want to combine stats from all post_install childrens.
-        return res
+        # Filter out results without values
+        res_arr = [r for r in res_arr if r['values']]
+        return res_arr
 
     @route(['/runbot/stats/<model("runbot.bundle"):bundle>/<model("runbot.trigger"):trigger>'], type='http', auth="public", website=True, sitemap=False)
     def modules_stats(self, bundle, trigger, search=None, **post):

runbot/models/build_stat.py

@@ -13,8 +13,8 @@ class BuildStat(models.Model):
 
     _sql_constraints = [
         (
-            "build_config_key_unique",
+            "category_build_config_step_unique_key",
-            "unique (build_id, config_step_id, category)",
+            "unique (category, build_id, config_step_id)",
             "Build stats must be unique for the same build step",
         )
     ]
@@ -23,5 +23,5 @@ class BuildStat(models.Model):
     config_step_id = fields.Many2one(
         "runbot.build.config.step", "Step", ondelete="cascade"
     )
-    category = fields.Char("Category", index=True)
+    category = fields.Char("Category")
     values = JsonDictField("Value")

runbot/static/src/stats/cache.js

@@ -0,0 +1,29 @@
+/** @odoo-module **/
+
+
+/**
+ * @typedef Bundle
+ *
+ * @property {Number} id
+ * @property {String} name
+ */
+
+const bundleNameCache = {}; // id to name
+
+/**
+ * Returns the name of the bundle according to the cache.
+ *
+ * @param {Number} bundleId id of the bundle
+ */
+export const getBundleName = (bundleId) => {
+    return bundleNameCache[bundleId] || bundleId.toString();
+}
+
+/**
+ * Populates the cache when a new list of bundle is loaded.
+ *
+ * @param {Bundle[]} bundles list of bundles
+ */
+export const populateCache = (bundles) => {
+    bundles.forEach(({ id, name }) => bundleNameCache[id] = name);
+}

runbot/static/src/stats/stats_chart.js

@@ -5,8 +5,17 @@ import { Component, useEffect, useRef, useState } from '@odoo/owl';
 import { debounce, filterKeys, randomColor } from '@runbot/utils';
 import { useConfig, onConfigChange } from '@runbot/stats/use_config';
 import { Chart } from '@runbot/chartjs';
+import { getBundleName } from '@runbot/stats/cache';
 
 
+/**
+ * @typedef StatsQueryResult
+ *
+ * @property {Number} id id of the build
+ * @property {Object.<string, number>} values object with value name as key and value as value
+ * @property {String} create_date creation date of the build
+ * @property {Number} bundle_id bundle id of the build
+ */
 export class StatsChart extends Component {
     static template = 'runbot.StatsChart';
     static props = {
@@ -39,6 +48,7 @@ export class StatsChart extends Component {
                 },
                 scales: {
                     x: {
+                        type: 'time',
                         display: true,
                         scaleLabel: {
                             display: true,
@@ -58,11 +68,16 @@ export class StatsChart extends Component {
                     if (activeElements.length === 0) {
                         return;
                     }
-                    const build_id = this.chartConfig.data.labels[activeElements[0].index];
+                    const { datasetIndex, index } = activeElements[0];
+                    const queryResult = this.chartConfig.data.datasets[datasetIndex].data[index]._queryResult;
+                    if (!queryResult) {
+                        return console.error('Queryresult not present for datasetIndex and index ', datasetIndex, index);
+                    }
+                    const { id } = queryResult;
                     if (shiftKey) {
-                        this.config.center_build_id = build_id;
+                        this.config.center_build_id = id;
                     } else {
-                        window.open(`/runbot/build/stats/${build_id}`);
+                        window.open(`/runbot/build/stats/${id}`);
                     }
                 }
             },
@@ -142,89 +157,132 @@ export class StatsChart extends Component {
         const {
             display_aggregate: aggregate,
             mode,
+            nb_dataset,
         } = this.config;
-        const { data } = this.state;
+
-        const builds = Object.keys(data);
+        /** @type {StatsQueryResult[]} */
-        const newestBuildStats = data[builds[builds.length - 1]];
+        const data = this.state.data;
-        const oldestBuildStats = data[builds[0]];
+        const bundles = new Set(data.map(({bundle_id}) => bundle_id));
-        const keys = Object.keys(newestBuildStats);
+        const hasMultiBundle = bundles.size > 1;
-        let idx = keys.indexOf('Aggregate Sum');
+        /**
-        if (aggregate === 'sum' && idx === -1) {
+         * Gets the label (group by key) for the given queryResult
-            keys.push('Aggregate Sum');
+         *
-            Object.values(data).forEach((buildData) => {
+         * @param {StatsQueryResult} queryResult
-                buildData['Aggregate Sum'] = Object.values(buildData).reduce((a, b) => a + b, 0);
+         * @param {String} valueName
-            });
+         */
-        } else if (aggregate !== 'sum' && idx !== -1) {
+        const getLabel = (queryResult, valueName) => {
-            keys.splice(idx, 1);
+            // Q: Change to bundleName if not main bundle?
+            if (hasMultiBundle) {
+                return `${valueName} (${getBundleName(queryResult.bundle_id)})`;
+            }
+            return `${valueName}`;
         }
-        idx = keys.indexOf('Aggregate Average');
+        // Group values by (valueName, bundle Id) we want to separate bundle ids.
-        if (aggregate === 'average' && idx === -1) {
+        const datasets = Object.values(data.reduce((agg, queryResult) => {
-            keys.push('Aggregate Average');
+            Object.entries(queryResult.values).forEach(([valueName, value]) => {
-            Object.values(data).forEach((buildData) => {
+                const label = getLabel(queryResult, valueName);
-                buildData['Aggregate Average'] = (Object.values(buildData).reduce((a, b) => a + b, 0) / Object.values(buildData).length);
+                if (!(agg[label])) {
-            });
+                    agg[label] = {
-        } else if (aggregate !== 'average' && idx !== -1) {
+                        label,
-            keys.splice(idx, 1);
+                        data: [],
-        }
+                        borderColor: randomColor(label),
-        // Mapping of keys to their sort value
+                        backgroundColor: 'rgba(0, 0, 0, 0)',
-        const sortValues = keys.reduce(
+                        lineTension: 0,
-            (dict, key) => {
+                        hidden: false,
-                const getValue = () => {
+                        _queryResult: queryResult,
-                    if (mode === 'normal') {
-                        return newestBuildStats[key];
-                    } else if (mode === 'alpha') {
-                        return key;
-                    } else if (mode === 'change_count') {
-                        return builds.reduce((agg, build, buildIdx) => {
-                            const currentBuild = data[build];
-                            const current = currentBuild[key];
-                            const previous = buildIdx === 0 ? undefined : data[builds[buildIdx - 1]][key];
-                            if (previous !== undefined && current !== undefined && previous != current) {
-                                agg += 1;
-                            }
-                            return agg;
-                        }, 0);
-                    } else if (mode === 'difference') {
-                        return Math.abs(
-                            newestBuildStats[key] - (oldestBuildStats[key] || 0)
-                        );
                     }
                 }
-                dict[key] = getValue();
+                agg[label].data.push({
-                return dict;
+                    x: queryResult.create_date,
-            }, {},
+                    y: value,
-        );
+                    _queryResult: queryResult,
-        keys.sort((k1, k2) => {
+                });
-            if (mode === 'alpha') {
+            })
-                return sortValues[k1].localeCompare(sortValues[k2]);
+            return agg;
+        }, {}));
+
+        // Compute selected aggregate
+        if (aggregate != 'none') {
+            const queryResultsByBundle = data.reduce((agg, queryResult) => {
+                if (!(agg[queryResult.bundle_id])) {
+                    agg[queryResult.bundle_id] = [];
+                }
+                agg[[queryResult.bundle_id]].push(queryResult);
+                return agg
+            }, {});
+            Object.values(queryResultsByBundle).forEach((queryResults) => {
+                const newData = queryResults.map((qs) => {
+                    return {
+                        x: qs.create_date,
+                        y: Object.values(qs.values).reduce((s, v) => s + v, 0),
+                        _queryResult: qs,
+                    }
+                });
+                let label = getLabel(datasets[0]._queryResult, 'Aggregate Sum');
+                if (aggregate === 'average') {
+                    label = getLabel(datasets[0]._queryResult, 'Aggregate Average');
+                    newData.forEach(d => d.y /= Object.values(d._queryResult.values).length);
+                }
+                datasets.push({
+                    label,
+                    data: newData,
+                    borderColor: randomColor(label),
+                    backgroundColor: 'rgba(0, 0, 0, 0)',
+                    lineTension: 0,
+                    hidden: false,
+                    _queryResult: datasets[0]._queryResult,
+                });
+            });
+        }
+        // Compute a sorting value for each dataset, sort
+        // Also recompute data if mode requires it.
+        datasets.forEach((dataset) => {
+            const getSortValue = () => {
+                if (mode === 'normal') {
+                    return dataset.data[0].y;
+                } else if (mode === 'alpha') {
+                    return dataset.label;
+                } else if (mode === 'change_count') {
+                    return dataset.data.reduce((agg, {y}, dataIdx) => {
+                        const previous = dataIdx === 0 ? undefined : dataset.data[dataIdx - 1].y;
+                        if (previous !== undefined && y !== undefined && previous != y) {
+                            agg += 1;
+                        }
+                        return agg;
+                    }, 0);
+                } else if (mode === 'difference') {
+                    return Math.abs(
+                        dataset.data[dataset.data.length - 1].y - dataset.data[0].y
+                    );
+                }
+            }
+            dataset._sortValue = getSortValue();
+            if (mode === 'change_count' || mode === 'difference') {
+                const firstValue = dataset.data[0].y;
+                dataset.data = dataset.data.map(d => {
+                    return {
+                        ...d,
+                        y: d.y - firstValue,
+                    };
+                });
             }
-            return sortValues[k2] - sortValues[k1]
         });
+        datasets.sort((ds1, ds2) => {
+            if (mode === 'alpha') {
+                return ds1._sortValue.localeCompare(ds2._sortValue);
+            }
+            return ds2._sortValue - ds1._sortValue;
+        });
+        // Change visibility of datasets according to config
         let visibleKeys;
-        if (this.config.nb_dataset !== -1) {
+        if (nb_dataset !== -1) {
-            visibleKeys = new Set(keys.slice(0, this.config.nb_dataset));
+            visibleKeys = new Set(datasets.slice(0, nb_dataset).map(ds => ds.label));
         } else {
             visibleKeys = new Set(this.config.getVisibleKeys());
         }
-        const getDisplayValue = (key, build) => {
+        datasets.forEach(ds => ds.hidden = !visibleKeys.has(ds.label));
-            if (build[key] === undefined) {
-                return NaN;
-            }
-            if (mode === 'normal' || mode === 'alpha') {
-                return build[key];
-            }
-            return build[key] - (oldestBuildStats[key] || 0)
-        }
         this.chartConfig.data = {
-            labels: builds,
+            datasets,
-            datasets: keys.map((key) => ({
-                label: key,
-                data: builds.map(build => getDisplayValue(key, data[build])),
-                borderColor: randomColor(key),
-                backgroundColor: 'rgba(0, 0, 0, 0)',
-                lineTension: 0,
-                hidden: !visibleKeys.has(key),
-            })),
         };
     }
 

runbot/static/src/stats/stats_chart.xml

@@ -16,11 +16,11 @@
         <div class="col-4">
           <label for="nb_dataset" class="form-label fw-bold mb-0">Display</label>
           <select class="form-select" id="nb_dataset" aria-label="Number Of Builds" t-on-change="(ev) => this.onChangeNbDataset(ev)">
-            <option value="-1">Custom</option>
+            <option value="-1" t-att-selected="config.nb_dataset === -1 ? 'selected' : undefined">Custom</option>
-            <option value="0">0</option>
+            <option value="0" t-att-selected="config.nb_dataset === 0 ? 'selected' : undefined">0</option>
-            <option value="10">Top 10</option>
+            <option value="10" t-att-selected="config.nb_dataset === 10 ? 'selected' : undefined">Top 10</option>
-            <option value="20">Top 20</option>
+            <option value="20" t-att-selected="config.nb_dataset === 20 ? 'selected' : undefined">Top 20</option>
-            <option value="50">Top 50</option>
+            <option value="50" t-att-selected="config.nb_dataset === 50 ? 'selected' : undefined">Top 50</option>
           </select>
         </div>
         <div class="col-4">
@@ -62,7 +62,7 @@
       </div>
       <div class="col-xs-3 col-md-2">
         <!-- legend -->
-        <div class="chart-legend" t-if="chartConfig.data?.datasets">
+        <div class="chart-legend overflow-x-auto" t-if="chartConfig.data?.datasets">
           <ul>
             <li t-foreach="chartConfig.data?.datasets" t-as="dataset" t-key="dataset.label" t-att-class="dataset.hidden ? 'disabled' : undefined"
               t-on-click.stop.prevent="() => this.onClickLegendItem(dataset.label)"

runbot/static/src/stats/stats_config.js

@@ -3,6 +3,7 @@
 import { Component, useEffect, useState } from '@odoo/owl';
 
 import { useConfig } from '@runbot/stats/use_config';
+import { getBundleName, populateCache } from '@runbot/stats/cache';
 import { debounce, randomColor } from '@runbot/utils';
 
 
@@ -13,26 +14,6 @@ import { debounce, randomColor } from '@runbot/utils';
  * @property {String} name
  */
 
-const bundleNameCache = {}; // id to name
-
-/**
- * Returns the name of the bundle according to the cache.
- *
- * @param {Number} bundleId id of the bundle
- */
-const getBundleName = (bundleId) => {
-    return bundleNameCache[bundleId] || bundleId.toString();
-}
-
-/**
- * Populates the cache when a new list of bundle is loaded.
- *
- * @param {Bundle[]} bundles list of bundles
- */
-const populateCache = (bundles) => {
-    bundles.forEach(({ id, name }) => bundleNameCache[id] = name);
-}
-
 export class StatsConfig extends Component {
     static template = 'runbot.StatsConfig';
     static props = {

runbot/static/src/stats/stats_root.js

@@ -7,6 +7,7 @@ import { StatsConfig } from '@runbot/stats/stats_config';
 import { StatsChart } from '@runbot/stats/stats_chart';
 import { useConfig } from '@runbot/stats/use_config';
 import { UrlUpdater } from '@runbot/stats/url_updater';
+import { populateCache } from '@runbot/stats/cache';
 
 
 export class StatsRoot extends Component {
@@ -54,6 +55,7 @@ export class StatsRoot extends Component {
     setup() {
         // Initialize shared configuration for children components.
         useConfig(false);
+        populateCache([this.props.bundle]);
     }
 }
 

runbot/templates/build_stats.xml

@@ -77,6 +77,8 @@
         <div id="wrapwrap">This page requires javascript to load</div>
       </t>
       <script type="text/javascript" src="/web/static/lib/Chart/Chart.js"></script>
+      <script type="text/javascript" src="/web/static/lib/luxon/luxon.js"/>
+      <script type="text/javascript" src="/web/static/lib/chartjs-adapter-luxon/chartjs-adapter-luxon.js"/>
       <t t-call-assets="runbot.assets_stats"/>
     </template>
   </data>