wip

runbot/__manifest__.py

@@ -32,6 +32,7 @@
         'templates/bundle.xml',
         'templates/commit.xml',
         'templates/dashboard.xml',
+        'templates/error_merge.xml',
         'templates/frontend.xml',
         'templates/git.xml',
         'templates/nginx.xml',
@@ -40,6 +41,7 @@
         'views/branch_views.xml',
         'views/build_error_link_views.xml',
         'views/build_error_views.xml',
+        'views/build_error_merge_views.xml',
         'views/build_views.xml',
         'views/bundle_views.xml',
         'views/codeowner_views.xml',

runbot/controllers/__init__.py

@@ -3,3 +3,4 @@
 from . import frontend
 from . import hook
 from . import badge
+from . import errors

runbot/controllers/errors.py

@@ -0,0 +1,11 @@
+from odoo.http import Controller, Response, request, route
+
+
+class ErrorControlle(Controller):
+
+    @route('/runbot/error/merge/result/<filter_id>', type='http', auth='public', website=True)
+    def error_filter_result(self, filter_id=None, **kwargs):
+        merger = request.env['runbot.build.error.merge'].browse(int(filter_id))
+        if not merger:
+            return Response('Error merge not found', status=404)
+        return request.render('runbot.error_merge_result', {'merger': merger, 'results': merger._get_matching_groups()})

runbot/models/__init__.py

@@ -6,6 +6,7 @@ from . import build
 from . import build_config
 from . import build_config_codeowner
 from . import build_error
+from . import build_error_merge
 from . import bundle
 from . import codeowner
 from . import commit

runbot/models/build_error.py

@@ -232,8 +232,8 @@ class BuildError(models.Model):
         for record in self:
             if record.common_qualifiers:
                 query = SQL(
-                    r"""SELECT id FROM runbot_build_error WHERE id != %s AND common_qualifiers @> %s""",
-                    record.id,
+                    r"""SELECT id FROM runbot_build_error WHERE id not in %s AND common_qualifiers @> %s""",
+                    tuple(record.ids),
                     json.dumps(record.common_qualifiers.dict),
                 )
                 self.env.cr.execute(query)
@@ -246,8 +246,8 @@ class BuildError(models.Model):
         for record in self:
             if record.common_qualifiers:
                 query = SQL(
-                    r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""",
-                    record.id,
+                    r"""SELECT id FROM runbot_build_error_content WHERE error_id not in %s AND qualifiers @> %s""",
+                    tuple(record.ids),
                     json.dumps(record.common_qualifiers.dict),
                 )
                 self.env.cr.execute(query)
@@ -260,8 +260,8 @@ class BuildError(models.Model):
         for record in self:
             if record.common_qualifiers:
                 query = SQL(
-                    r"""SELECT id FROM runbot_build_error WHERE id != %s AND unique_qualifiers @> %s""",
-                    record.id,
+                    r"""SELECT id FROM runbot_build_error WHERE id not in %s AND unique_qualifiers @> %s""",
+                    tuple(record.ids),
                     json.dumps(record.unique_qualifiers.dict),
                 )
                 self.env.cr.execute(query)
@@ -274,8 +274,8 @@ class BuildError(models.Model):
         for record in self:
             if record.common_qualifiers:
                 query = SQL(
-                    r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""",
-                    record.id,
+                    r"""SELECT id FROM runbot_build_error_content WHERE error_id not in %s AND qualifiers @> %s""",
+                    tuple(record.ids),
                     json.dumps(record.unique_qualifiers.dict),
                 )
                 self.env.cr.execute(query)
@@ -537,6 +537,9 @@ class BuildError(models.Model):
         existing_errors_contents = self.env['runbot.build.error.content'].search([('fingerprint', 'in', list(hash_dict.keys())), ('error_id.active', '=', True)])
         existing_fingerprints = {error.fingerprint: error for error in existing_errors_contents}
         build_error_contents |= existing_errors_contents
+
+        # TODO
+        error_content_merge = self.env['runbot.build.error.merge'].search([('auto_merge', '=', True), ('active', '=', True)])
         # create an error for the remaining entries
         for fingerprint, logs in hash_dict.items():
             if fingerprint in existing_fingerprints:
@@ -546,13 +549,24 @@ class BuildError(models.Model):
                     error.metadata = logs[0].metadata
 
                 continue
-            new_build_error_content = self.env['runbot.build.error.content'].create({
+            vals = {
+                'error_id': None,
                 'content': logs[0].message,
                 'module_name': logs[0].name.removeprefix('odoo.').removeprefix('addons.'),
                 'file_path': logs[0].path,
                 'function': logs[0].func,
                 'metadata': logs[0].metadata,
-            })
+                'canonical_tag': logs[0].metadata.get('test', {}).get('canonical_tag')
+            }
+            self._qualify(vals)  # populate vals with qualifiers
+            similar_domain = error_content_merge._get_similar_domain(vals)
+            error_candidates = self.env['runbot.build.error.content'].search(similar_domain)
+            if error_candidates:
+                vals['error_id'] = error_candidates[0].error_id.id
+
+            new_build_error_content = self.env['runbot.build.error.content'].create()
+
+
             build_error_contents |= new_build_error_content
             existing_fingerprints[fingerprint] = new_build_error_content
 
@@ -595,6 +609,7 @@ class BuildErrorContent(models.Model):
     _inherit = ('mail.thread', 'mail.activity.mixin', 'runbot.build.error.seen.mixin')
     _rec_name = "id"
 
+    active = fields.Boolean('Active', related='error_id.active')
     error_id = fields.Many2one('runbot.build.error', 'Linked to', index=True, required=True)
     error_display_id = fields.Integer(compute='_compute_error_display_id', string="Error id")
     content = fields.Text('Error message', required=True)
@@ -614,7 +629,7 @@ class BuildErrorContent(models.Model):
     version_ids = fields.One2many('runbot.version', compute='_compute_version_ids', string='Versions', search='_search_version')
     trigger_ids = fields.Many2many('runbot.trigger', compute='_compute_trigger_ids', string='Triggers', search='_search_trigger_ids')
     tag_ids = fields.Many2many('runbot.build.error.tag', string='Tags')
-    qualifiers = JsonDictField('Qualifiers', index=True)
+    qualifiers = JsonDictField('Qualifiers', index=True, store=True, compute="_compute_qualifiers")
     similar_ids = fields.One2many('runbot.build.error.content', compute='_compute_similar_ids')
 
     responsible = fields.Many2one(related='error_id.responsible')
@@ -716,8 +731,8 @@ class BuildErrorContent(models.Model):
         for record in self:
             if record.qualifiers:
                 query = SQL(
-                    r"""SELECT id FROM runbot_build_error_content WHERE id != %s AND qualifiers @> %s AND qualifiers <@ %s""",
-                    record.id,
+                    r"""SELECT id FROM runbot_build_error_content WHERE id not in %s AND qualifiers @> %s AND qualifiers <@ %s""",
+                    tuple(record.ids),
                     json.dumps(record.qualifiers.dict),
                     json.dumps(record.qualifiers.dict),
                 )
@@ -726,6 +741,10 @@ class BuildErrorContent(models.Model):
             else:
                 record.similar_ids = False
 
+    @api.depends('content', 'canonical_tag', 'module_name', 'file_path', 'function')
+    def _compute_qualifiers(self):
+        self._qualify()
+
     @api.model
     def _digest(self, s):
         """
@@ -779,16 +798,16 @@ class BuildErrorContent(models.Model):
         domain = [('id', 'in', self.ids)] if self else []
         return [r[1] for r in self._read_group(domain, ('fingerprint'), ('id:array_agg'), [('id:count', '>', 1)])]
 
-    def _qualify(self):
+    def _qualify(self, vals):
         qualify_regexes = self.env['runbot.error.qualify.regex'].search([])
-        for record in self:
+        for record in self or vals:
             all_qualifiers = {}
             for qualify_regex in qualify_regexes:
                 res = qualify_regex._qualify(record)
                 if res:
                     # res.update({'qualifier_id': qualify_regex.id}) Probably not a good idea
                     all_qualifiers.update(res)
-            record.qualifiers = all_qualifiers
+            record['qualifiers'] = all_qualifiers
 
     ####################
     #   Actions

runbot/models/build_error_merge.py

@@ -0,0 +1,71 @@
+from odoo import models, fields, api
+from odoo.osv import expression
+
+class BuildErrorMerge(models.Model):
+    _name = 'runbot.build.error.merge'
+    _description = 'Error Merge patterns'
+    _inherit = ['mail.thread']
+
+    active = fields.Boolean('Active', default=True)
+    name = fields.Char('Name', required=True)
+    merge_filter_ids = fields.One2many('runbot.build.error.merge.filters', 'error_merge_id', 'Merge Lines')
+    description = fields.Char('Description', compute='_compute_description', store=True, tracking=True)
+    oneline_description = fields.Char('One Line Description', compute='_compute_description_online')
+    auto_merge = fields.Boolean('Auto Merge', default=False)
+
+    def _get_read_group_params(self):
+        domain = [('active', '=', True)]
+        for filter in self.merge_filter_ids:
+            domain = expression.AND([domain, [(filter.field_name, '!=', False)]])
+        groups = self.merge_filter_ids.mapped('field_name')
+        assert groups
+
+        return (
+            domain,
+            groups,
+        )
+
+    def _get_matching_groups(self):
+        domain, groups = self._get_read_group_params()
+        result = self.env['runbot.build.error.content']._read_group(
+            domain,
+            groups,
+            ['id:array_agg'],
+            [('error_id:count_distinct', '>', 1)]
+        )
+        return result
+
+    def _get_similar_domain(self, error_content):
+        result = [expression.FALSE_LEAF]
+        for record in self:
+            if all(error_content[f.field_name] for f in record.merge_filter_ids):
+                merge_domain = [(f.field_name, '==', error_content[f.field_name]) for f in  record.merge_filter_ids]
+                result = expression.OR([result, merge_domain])
+        return result
+
+    def action_see_matches(self):
+        self.ensure_one()
+        return {
+            'name': 'Error Candidates',
+            'type': 'ir.actions.act_url',
+            'url': f"/runbot/error/merge/result/{self.id}",
+        }
+
+    @api.depends('merge_filter_ids.field_name')
+    def _compute_description(self):
+        for record in self:
+            record.description = '\n'.join(f.field_name for f in record.merge_filter_ids)
+
+    @api.depends('description')
+    def _compute_description_online(self):
+        for record in self:
+            record.oneline_description = record.description.replace('\n', ', ')
+
+
+class BuildErrorMergeFilter(models.Model):
+    _name = 'runbot.build.error.merge.filters'
+    _description = 'Error Merge patterns filters'
+
+    field = fields.Many2one('ir.model.fields', 'Field', domain=[('model_id.model', '=', 'runbot.build.error.content')], required=True, ondelete='cascade')
+    field_name = fields.Char('Field Name', related='field.name', store=True, readonly=True)
+    error_merge_id = fields.Many2one('runbot.build.error.merge', 'Error Merge', required=True)

runbot/security/ir.model.access.csv

@@ -151,4 +151,9 @@ access_runbot_build_stat_regex_wizard,access_runbot_build_stat_regex_wizard,mode
 
 access_runbot_host_message,access_runbot_host_message,runbot.model_runbot_host_message,runbot.group_runbot_admin,1,0,0,0
 
+access_runbot_build_error_merge,access_runbot_build_error_merge,runbot.model_runbot_build_error_merge,base.group_user,1,0,0,0
+access_runbot_build_error_merge_filters,access_runbot_build_error_merge_filters,runbot.model_runbot_build_error_merge_filters,base.group_user,1,0,0,0
+access_runbot_build_error_merge,access_runbot_build_error_merge,runbot.model_runbot_build_error_merge,runbot.group_runbot_admin,1,1,1,1
+access_runbot_build_error_merge_filters,access_runbot_build_error_merge_filters,runbot.model_runbot_build_error_merge_filters,runbot.group_runbot_admin,1,1,1,1
+
 

runbot/templates/error_merge.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+    <data>
+      <template id="runbot.error_merge_result">
+        <t t-call='runbot.layout'>
+            <t t-set="title">Error Merge</t>
+            <h3>Error merge rule: <t t-esc="merger.name"/> (<t t-esc="len(results)"/>)</h3>
+            <div class="row">
+                <t t-foreach="results" t-as="result">
+                    <div class="col-md-12">
+                        <div class="card">
+                            <div class="card-header">
+                                <span><t t-esc="'|'.join(result[:-1])"/></span><button class="btn btn-primary float-end" onclick="merge()">Merge (TODO)</button>
+                            </div>
+                            <div class="card-body">
+                                <table class="table">
+                                <t t-foreach="env['runbot.build.error.content'].browse(result[-1])" t-as="error_content">
+                                    <tr>
+                                        <td><t t-esc="error_content.error_id.active"/></td> 
+                                        <td><a t-attf-href="/odoo/error_content/{{error_content.id}}"><t t-esc="error_content.id"/></a></td> 
+                                        <td><a t-attf-href="/odoo/error/{{error_content.error_id.id}}"><t t-esc="error_content.error_id.id"/></a></td> 
+                                        <td>
+                                            <span data-bs-toggle="collapse" t-attf-data-bs-target="#collapse{{error_content.id}}"><t t-esc="error_content.content.split('\n')[0]"/></span>
+                                            <pre t-attf-id="collapse{{error_content.id}}" class="collapse"><t t-esc="error_content.content"/></pre>
+                                        </td>
+                                    </tr>
+                                </t>
+                                </table>
+                            </div>
+                        </div>
+                    </div>
+                </t>
+            </div>
+        </t>
+      </template>
+    </data>
+</odoo>

runbot/tests/__init__.py

@@ -1,5 +1,6 @@
 from . import common
 from . import test_batch
+from . import test_build_error_merge
 from . import test_repo
 from . import test_build_error
 from . import test_branch

runbot/tests/test_build_error_merge.py

@@ -0,0 +1,9 @@
+from .common import RunbotCase
+
+
+
+class TestErrorMerge(self):
+
+    def test_auto_merge(self):
+
+    def test_inactive_no_auto_merge(self):

runbot/views/build_error_merge_views.xml

@@ -0,0 +1,44 @@
+<odoo>
+  <data>
+    <record id="build_error_merge_view_form" model="ir.ui.view">
+        <field name="name">runbot.build.error.merge.form</field>
+        <field name="model">runbot.build.error.merge</field>
+        <field name="arch" type="xml">
+          <form>
+            <header>
+              <button name="action_see_matches" string="See matches" type="object" class="oe_highlight"/>
+            </header>
+            <sheet>
+              <group>
+                <field name="name"/>
+                <field name="auto_merge"/>
+                <field name="merge_filter_ids">
+                    <list editable="bottom">
+                        <field name="field"/>
+                        <field name="field_name"/>
+                    </list>
+                </field>
+              </group>
+            </sheet>
+            <chatter/>
+          </form>
+        </field>
+    </record>
+    <record id="build_error_merge_view_list" model="ir.ui.view">
+        <field name="name">runbot.build.error.merge.list</field>
+        <field name="model">runbot.build.error.merge</field>
+        <field name="arch" type="xml">
+            <list string="Error merge">
+                <field name="name"/>
+                <field name="auto_merge"/>
+                <field name="oneline_description"/>
+            </list>
+        </field>
+    </record>
+    <record id="open_build_error_merge_views" model="ir.actions.act_window">
+        <field name="name">Error merge</field>
+        <field name="res_model">runbot.build.error.merge</field>
+        <field name="view_mode">list,form</field>
+    </record>
+  </data>
+</odoo>

runbot/views/menus.xml

@@ -38,6 +38,8 @@
     <menuitem name="Errors" id="runbot_menu_build_error_tree" parent="runbot_menu_manage_errors" sequence="5" action="open_view_build_error_tree"/>
     <menuitem name="Errors contents" id="runbot_menu_build_error_content_tree" parent="runbot_menu_manage_errors" sequence="10" action="open_view_build_error_content_tree"/>
     <menuitem name="Errors Qualifying" id="runbot_menu_build_error_qualify_regex_tree" parent="runbot_menu_manage_errors" sequence="10" action="open_view_build_error_qualify_regex_tree"/>
+    <menuitem name="Error regex" id="runbot_menu_error_regex_tree" parent="runbot_menu_manage_errors" sequence="20" action="open_view_error_regex"/>
+    <menuitem name="Error auto merge" id="runbot_menu_error_auto_merge" parent="runbot_menu_manage_errors" sequence="25" action="open_build_error_merge_views"/>
 
     <menuitem name="Teams" id="runbot_menu_teams" parent="runbot_menu_root" sequence="1000"/>
     <menuitem name="Teams" id="runbot_menu_team_tree" parent="runbot_menu_teams" sequence="30" action="open_view_runbot_team"/>
@@ -59,7 +61,7 @@
     <menuitem id="runbot_menu_upgrade_regex_tree" parent="menu_runbot_settings" sequence="60" action="open_view_upgrade_regex_tree"/>
     <menuitem name="Stats Regexes" id="runbot_menu_stat" parent="menu_runbot_settings" sequence="70" action="open_view_stat_regex_tree"/>
     <menuitem name="Stat Regex Wizard" id="runbot_menu_stat_wizard" parent="menu_runbot_settings" sequence="80" action="runbot_stat_regex_wizard_action"/>
-    <menuitem name="Error regex" id="runbot_menu_error_regex_tree" parent="menu_runbot_settings" sequence="20" action="open_view_error_regex"/>
+
     
     <menuitem name="Technical" id="runbot_menu_technical" parent="menu_runbot_settings" sequence="10000" groups="base.group_system"/>
     <menuitem id="runbot_menu_ir_cron_act" action="base.ir_cron_act" parent="runbot_menu_technical"/>