[IMP] runbot: add common and unique qualifiers on build error

This commit adds a `common_qualifiers` field on build error. Its purpose
is mainly to find similary qualified errors and similary qualified error
contents. This field is computed by finding qualifiers in common in all
the qualifiers of the error contents linked to the error.

A new `unique_qualifiers` is also added for the same kind of puprpose.
This field is computed by finding non contradictory qualifiers of the
linked error contents.

The fields can be used in 4 tabs added on the build error form.

runbot/models/build_error.py

@@ -103,6 +103,13 @@ class BuildError(models.Model):
     tags_min_version_id = fields.Many2one('runbot.version', 'Tags Min version', help="Minimal version where the test tags will be applied.")
     tags_max_version_id = fields.Many2one('runbot.version', 'Tags Max version', help="Maximal version where the test tags will be applied.")
 
+    common_qualifiers = JsonDictField('Common Qualifiers', compute='_compute_common_qualifiers', store=True, help="Minimal qualifiers in common needed to link error content.")
+    similar_ids = fields.One2many('runbot.build.error', compute='_compute_similar_ids', string="Similar Errors", help="Similar Errors based on common qualifiers")
+    similar_content_ids = fields.One2many('runbot.build.error.content', compute='_compute_similar_content_ids', string="Similar Error Contents", help="Similar Error contents based on common qualifiers")
+    unique_qualifiers = JsonDictField('Non conflicting Qualifiers', compute='_compute_unique_qualifiers', store=True, help="Non conflicting qualifiers in common needed to link error content.")
+    analogous_ids = fields.One2many('runbot.build.error', compute='_compute_analogous_ids', string="Analogous Errors", help="Analogous Errors based on unique qualifiers")
+    analogous_content_ids= fields.One2many('runbot.build.error.content', compute='_compute_analogous_content_ids', string="Analogous Error Contents", help="Analogous Error contents based on unique qualifiers")
+
     # Build error related data
     build_error_link_ids = fields.Many2many('runbot.build.error.link', compute=_compute_related_error_content_ids('build_error_link_ids'), search=_search_related_error_content_ids('build_error_link_ids'))
     unique_build_error_link_ids = fields.Many2many('runbot.build.error.link', compute='_compute_unique_build_error_link_ids')
@@ -149,6 +156,84 @@ class BuildError(models.Model):
         for record in self:
             record.random = any(error.random for error in record.error_content_ids)
 
+    @api.depends('error_content_ids.qualifiers')
+    def _compute_common_qualifiers(self):
+        for record in self:
+            qualifiers = defaultdict(set)
+            key_count = defaultdict(int)
+            for content in record.error_content_ids:
+                for key, value in content.qualifiers.dict.items():
+                    qualifiers[key].add(value)
+                    key_count[key] += 1
+            record.common_qualifiers = {k: v.pop() for k, v in qualifiers.items() if len(v) == 1 and key_count[k] == len(record.error_content_ids)}
+
+    @api.depends('error_content_ids.qualifiers')
+    def _compute_unique_qualifiers(self):
+        for record in self:
+            qualifiers = defaultdict(set)
+            key_count = defaultdict(int)
+            for content in record.error_content_ids:
+                for key, value in content.qualifiers.dict.items():
+                    qualifiers[key].add(value)
+                    key_count[key] += 1
+            record.unique_qualifiers = {k: v.pop() for k, v in qualifiers.items() if len(v) == 1}
+
+    @api.depends('common_qualifiers')
+    def _compute_similar_ids(self):
+        for record in self:
+            if record.common_qualifiers:
+                query = SQL(
+                    r"""SELECT id FROM runbot_build_error WHERE id != %s AND common_qualifiers @> %s""",
+                    record.id,
+                    json.dumps(record.common_qualifiers.dict),
+                )
+                self.env.cr.execute(query)
+                record.similar_ids = self.env['runbot.build.error'].browse([rec[0] for rec in self.env.cr.fetchall()])
+            else:
+                record.similar_ids = False
+
+    @api.depends('common_qualifiers')
+    def _compute_similar_content_ids(self):
+        for record in self:
+            if record.common_qualifiers:
+                query = SQL(
+                    r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""",
+                    record.id,
+                    json.dumps(record.common_qualifiers.dict),
+                )
+                self.env.cr.execute(query)
+                record.similar_content_ids = self.env['runbot.build.error.content'].browse([rec[0] for rec in self.env.cr.fetchall()])
+            else:
+                record.similar_content_ids = False
+
+    @api.depends('common_qualifiers')
+    def _compute_analogous_ids(self):
+        for record in self:
+            if record.common_qualifiers:
+                query = SQL(
+                    r"""SELECT id FROM runbot_build_error WHERE id != %s AND unique_qualifiers @> %s""",
+                    record.id,
+                    json.dumps(record.unique_qualifiers.dict),
+                )
+                self.env.cr.execute(query)
+                record.analogous_ids = self.env['runbot.build.error'].browse([rec[0] for rec in self.env.cr.fetchall()])
+            else:
+                record.analogous_ids = False
+
+    @api.depends('common_qualifiers')
+    def _compute_analogous_content_ids(self):
+        for record in self:
+            if record.common_qualifiers:
+                query = SQL(
+                    r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""",
+                    record.id,
+                    json.dumps(record.unique_qualifiers.dict),
+                )
+                self.env.cr.execute(query)
+                record.analogous_content_ids = self.env['runbot.build.error.content'].browse([rec[0] for rec in self.env.cr.fetchall()])
+            else:
+                record.analogous_content_ids = False
+
 
     @api.constrains('test_tags')
     def _check_test_tags(self):
@@ -202,6 +287,8 @@ class BuildError(models.Model):
                 if not error.team_id:
                     error.team_id = previous_error.team_id
             previous_error.error_content_ids.write({'error_id': self})
+            previous_error.common_qualifiers = dict()
+            previous_error.unique_qualifiers = dict()
             if not previous_error.test_tags:
                 previous_error.message_post(body=Markup('Error merged into %s') % error._get_form_link())
                 previous_error.active = False
@@ -245,6 +332,50 @@ class BuildError(models.Model):
             'target': 'current',
         }
 
+    def action_view_similary_qualified(self):
+        return {
+            'type': 'ir.actions.act_window',
+            'views': [(False, 'list'), (False, 'form')],
+            'res_model': 'runbot.build.error',
+            'domain': [('id', 'in', [self.id] + self.similar_ids.ids)],
+            'context': {'active_test': False},
+            'target': 'current',
+            'name': 'Similary Qualified Errors'
+        }
+
+    def action_view_similary_qualified_contents(self):
+        return {
+            'type': 'ir.actions.act_window',
+            'views': [(False, 'list'), (False, 'form')],
+            'res_model': 'runbot.build.error.content',
+            'domain': [('id', 'in', self.similar_content_ids.ids)],
+            'context': {'active_test': False},
+            'target': 'current',
+            'name': 'Similary Qualified Contents'
+        }
+
+    def action_view_analogous_qualified(self):
+        return {
+            'type': 'ir.actions.act_window',
+            'views': [(False, 'list'), (False, 'form')],
+            'res_model': 'runbot.build.error',
+            'domain': [('id', 'in', [self.id] + self.analogous_ids.ids)],
+            'context': {'active_test': False},
+            'target': 'current',
+            'name': 'Similary Qualified Errors'
+        }
+
+    def action_view_analogous_qualified_contents(self):
+        return {
+            'type': 'ir.actions.act_window',
+            'views': [(False, 'list'), (False, 'form')],
+            'res_model': 'runbot.build.error.content',
+            'domain': [('id', 'in', self.analogous_content_ids.ids)],
+            'context': {'active_test': False},
+            'target': 'current',
+            'name': 'Similary Qualified Contents'
+        }
+
     def action_assign(self):
         teams = None
         repos = None
@@ -731,7 +862,8 @@ for error_content in self:
         result = False
         if content and self.regex:
             result = re.search(self.regex, content, flags=re.MULTILINE)
-        return result.groupdict() if result else {}
+        # filtering empty values to allow non mandatory named groups
+        return {k:v for k,v in result.groupdict().items() if v} if result else {}
 
 
 class QualifyErrorTest(models.Model):

runbot/tests/test_build_error.py

@@ -529,6 +529,61 @@ class TestBuildError(RunbotCase):
         self.assertEqual([], self.BuildError._disabling_tags(build_fixing))
         self.assertEqual(['-bar'], self.BuildError._disabling_tags(build_random))
 
+    def test_build_error_qualifers(self):
+        error_contents = self.BuildErrorContent.create(
+            [
+                {
+                    "content": "Tour foobar_tour failed at step click_here in mode admin",
+                },
+                                {
+                    "content": "Tour foobar_tour failed at step click_here in mode demo",
+                },
+                {
+                    "content": "Tour foobar_tour failed -> click_here",
+                },
+                {
+                    "content": "Tour foobar_tour failed",
+                },
+            ]
+        )
+        self.assertEqual(len(error_contents), 4)
+        self.assertEqual(len(self.BuildError.search([('error_content_ids', 'in', error_contents.ids)])), 4)
+
+        self.env['runbot.error.qualify.regex'].create({
+            "regex": r"Tour (?P<tour_name>\w+) failed( at step (?P<tour_step>\w+) in mode (?P<tour_mode>\w+))?"
+        })
+        error_contents._qualify()
+
+        expected_common_qualifiers = {'tour_name': 'foobar_tour', 'tour_step': 'click_here'}
+        self.assertEqual(error_contents[0].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'admin'})
+        self.assertEqual(error_contents[1].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'demo'})
+        self.assertEqual(error_contents[2].qualifiers.dict, {'tour_name': 'foobar_tour'})
+
+        self.env['runbot.error.qualify.regex'].create({
+            "regex": r"Tour (?P<tour_name>\w+) failed -> (?P<tour_step>\w+)"
+        })
+
+        error_contents._qualify()
+        self.assertEqual(error_contents[0].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'admin'})
+        self.assertEqual(error_contents[1].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'demo'})
+        self.assertEqual(error_contents[2].qualifiers.dict, expected_common_qualifiers)
+
+        # now let's say that we merge admin and demo errors
+        main_error = error_contents[0].error_id
+        self.assertEqual(len(main_error.error_content_ids), 1)
+        main_error._merge(error_contents[1].error_id)
+        self.assertEqual(len(main_error.error_content_ids), 2)
+
+        self.assertEqual(main_error.common_qualifiers.dict, expected_common_qualifiers)
+        self.env.flush_all()
+        self.assertEqual(len(main_error.similar_ids), 1)
+        self.assertEqual(main_error.similar_ids[0], error_contents[2].error_id)
+
+        # let's merge all errors and verify the qualifiers
+        main_error._merge(error_contents.error_id)
+        self.assertEqual(main_error.common_qualifiers.dict, {'tour_name': 'foobar_tour'})
+        self.assertEqual(main_error.unique_qualifiers.dict, {'tour_name': 'foobar_tour', 'tour_step': 'click_here'})
+
     def test_build_error_team_wildcards(self):
         website_team = self.RunbotTeam.create({
             'name': 'website_test',

runbot/views/build_error_views.xml

@@ -53,7 +53,7 @@
               </group>
               <group>
                 <field name="previous_error_id" readonly="1" invisible="not previous_error_id" text-decoration-danger="True"/>
-              </group>  
+              </group>
               <notebook>
                 <page string="Builds">
                   <field name="unique_build_error_link_ids" widget="one2many" options="{'not_delete': True, 'no_create': True}" readonly="1">
@@ -82,6 +82,124 @@
                     </list>
                   </field>
                 </page>
+                <page string="Similary qualified Errors" groups="runbot.group_runbot_admin,runbot.group_runbot_error_manager" invisible="not common_qualifiers">
+                  <div class="btn-group">
+                    <button class="btn btn-sm btn-primary" type="object" name="action_view_similary_qualified">
+                      View similary qualified errors
+                    </button>
+                  </div>
+                  <field name="common_qualifiers" widget="runbotjsonb"/>
+                  <field name="similar_ids" widget="one2many" options="{'not_delete': True, 'no_create': True}" readonly="1">
+                    <list decoration-danger="test_tags and (fixing_pr_alive or not fixing_pr_id)"
+                          decoration-success="fixing_pr_id and not test_tags and not fixing_pr_alive"
+                          decoration-warning="test_tags and fixing_pr_id and not fixing_pr_alive"
+                          >
+                      <field name="id"/>
+                      <field name="name" optional="show" readonly="1"/>
+                      <field name="description" optional="hide" readonly="1"/>
+                      <field name="first_seen_date" string="First Seen" optional="hide" readonly="1"/>
+                      <field name="last_seen_date" string="Last Seen" readonly="1" options="{'link_field': 'last_seen_build_id'}"/>
+                      <field name="last_seen_build_id" column_invisible="True"/>
+                      <field name="error_count" readonly="1"/>
+                      <field name="build_count" readonly="1"/>
+                      <field name="team_id"/>
+                      <field name="responsible" optional="show"/>
+                      <field name="test_tags" optional="hide"/>
+                      <field name="fixing_pr_id" optional="hide"/>
+                      <field name="fixing_pr_alive" optional="hide"/>
+                    </list>
+                  </field>
+                </page>
+                <page string="Similary qualified Error Contents" groups="runbot.group_runbot_admin,runbot.group_runbot_error_manager" invisible="not common_qualifiers">
+                  <div class="btn-group">
+                    <button class="btn btn-sm btn-primary" type="object" name="action_view_similary_qualified_contents">
+                      View Similary Contents
+                    </button>
+                  </div>
+                  <field name="common_qualifiers" widget="runbotjsonb"/>
+                  <field name="similar_content_ids" widget="one2many" options="{'not_delete': True, 'no_create': True}" readonly="1">
+                    <list string="Error Contents"
+                          decoration-danger="test_tags and (fixing_pr_alive or not fixing_pr_id)"
+                          decoration-success="fixing_pr_id and not test_tags and not fixing_pr_alive"
+                          decoration-warning="test_tags and fixing_pr_id and not fixing_pr_alive"
+                          >
+                        <field name="error_display_id" optional="show"/>
+                        <field name="module_name" optional="show" readonly="1"/>
+                        <field name="summary" optional="show" readonly="1"/>
+                        <field name="random" string="Random"/>
+                        <field name="first_seen_date" string="First Seen" optional="hide" readonly="1"/>
+                        <field name="last_seen_date" string="Last Seen" readonly="1"/>
+                        <field name="build_count" readonly="1"/>
+                        <field name="team_id"/>
+                        <field name="test_tags" optional="hide"/>
+                        <field name="tags_min_version_id" string="Tags Min" optional="hide"/>
+                        <field name="tags_max_version_id" string="Tags Max" optional="hide"/>
+                        <field name="fixing_pr_id" optional="hide"/>
+                        <field name="fixing_pr_alive" optional="hide"/>
+                        <field name="fixing_pr_url" widget="url" text="view PR" readonly="1" invisible="not fixing_pr_url"/>
+                        <field name="fingerprint" optional="hide"/>
+                      </list>
+                  </field>
+                </page>
+                <page string="Analogous Errors" groups="runbot.group_runbot_admin,runbot.group_runbot_error_manager" invisible="not unique_qualifiers">
+                  <div class="btn-group">
+                    <button class="btn btn-sm btn-primary" type="object" name="action_view_analogous_qualified">
+                      View analogous errors
+                    </button>
+                  </div>
+                  <field name="unique_qualifiers" widget="runbotjsonb"/>
+                  <field name="analogous_ids" widget="one2many" options="{'not_delete': True, 'no_create': True}" readonly="1">
+                    <list decoration-danger="test_tags and (fixing_pr_alive or not fixing_pr_id)"
+                          decoration-success="fixing_pr_id and not test_tags and not fixing_pr_alive"
+                          decoration-warning="test_tags and fixing_pr_id and not fixing_pr_alive"
+                          >
+                      <field name="id"/>
+                      <field name="name" optional="show" readonly="1"/>
+                      <field name="description" optional="hide" readonly="1"/>
+                      <field name="first_seen_date" string="First Seen" optional="hide" readonly="1"/>
+                      <field name="last_seen_date" string="Last Seen" readonly="1" options="{'link_field': 'last_seen_build_id'}"/>
+                      <field name="last_seen_build_id" column_invisible="True"/>
+                      <field name="error_count" readonly="1"/>
+                      <field name="build_count" readonly="1"/>
+                      <field name="team_id"/>
+                      <field name="responsible" optional="show"/>
+                      <field name="test_tags" optional="hide"/>
+                      <field name="fixing_pr_id" optional="hide"/>
+                      <field name="fixing_pr_alive" optional="hide"/>
+                    </list>
+                  </field>
+                </page>
+                <page string="Analogous Error Contents" groups="runbot.group_runbot_admin,runbot.group_runbot_error_manager" invisible="not unique_qualifiers">
+                  <div class="btn-group">
+                    <button class="btn btn-sm btn-primary" type="object" name="action_view_analogous_qualified_contents">
+                      View Analogous Contents
+                    </button>
+                  </div>
+                  <field name="unique_qualifiers" widget="runbotjsonb"/>
+                  <field name="analogous_content_ids" widget="one2many" options="{'not_delete': True, 'no_create': True}" readonly="1">
+                    <list string="Error Contents"
+                          decoration-danger="test_tags and (fixing_pr_alive or not fixing_pr_id)"
+                          decoration-success="fixing_pr_id and not test_tags and not fixing_pr_alive"
+                          decoration-warning="test_tags and fixing_pr_id and not fixing_pr_alive"
+                          >
+                        <field name="error_display_id" optional="show"/>
+                        <field name="module_name" optional="show" readonly="1"/>
+                        <field name="summary" optional="show" readonly="1"/>
+                        <field name="random" string="Random"/>
+                        <field name="first_seen_date" string="First Seen" optional="hide" readonly="1"/>
+                        <field name="last_seen_date" string="Last Seen" readonly="1"/>
+                        <field name="build_count" readonly="1"/>
+                        <field name="team_id"/>
+                        <field name="test_tags" optional="hide"/>
+                        <field name="tags_min_version_id" string="Tags Min" optional="hide"/>
+                        <field name="tags_max_version_id" string="Tags Max" optional="hide"/>
+                        <field name="fixing_pr_id" optional="hide"/>
+                        <field name="fixing_pr_alive" optional="hide"/>
+                        <field name="fixing_pr_url" widget="url" text="view PR" readonly="1" invisible="not fixing_pr_url"/>
+                        <field name="fingerprint" optional="hide"/>
+                      </list>
+                  </field>
+                </page>
               </notebook>
           </sheet>
           <chatter/>