From 6fe80265ba3c38634c3fbbc2fc284a7fa1d7d527 Mon Sep 17 00:00:00 2001 From: Christophe Monniez Date: Thu, 23 Jan 2025 16:45:04 +0100 Subject: [PATCH] [IMP] runbot: add common and unique qualifiers on build error This commit adds a `common_qualifiers` field on build error. Its purpose is mainly to find similary qualified errors and similary qualified error contents. This field is computed by finding qualifiers in common in all the qualifiers of the error contents linked to the error. A new `unique_qualifiers` is also added for the same kind of puprpose. This field is computed by finding non contradictory qualifiers of the linked error contents. The fields can be used in 4 tabs added on the build error form. --- runbot/models/build_error.py | 134 ++++++++++++++++++++++++++++- runbot/tests/test_build_error.py | 55 ++++++++++++ runbot/views/build_error_views.xml | 120 +++++++++++++++++++++++++- 3 files changed, 307 insertions(+), 2 deletions(-) diff --git a/runbot/models/build_error.py b/runbot/models/build_error.py index 7bb96c78..f307330c 100644 --- a/runbot/models/build_error.py +++ b/runbot/models/build_error.py @@ -103,6 +103,13 @@ class BuildError(models.Model): tags_min_version_id = fields.Many2one('runbot.version', 'Tags Min version', help="Minimal version where the test tags will be applied.") tags_max_version_id = fields.Many2one('runbot.version', 'Tags Max version', help="Maximal version where the test tags will be applied.") + common_qualifiers = JsonDictField('Common Qualifiers', compute='_compute_common_qualifiers', store=True, help="Minimal qualifiers in common needed to link error content.") + similar_ids = fields.One2many('runbot.build.error', compute='_compute_similar_ids', string="Similar Errors", help="Similar Errors based on common qualifiers") + similar_content_ids = fields.One2many('runbot.build.error.content', compute='_compute_similar_content_ids', string="Similar Error Contents", help="Similar Error contents based on common qualifiers") + unique_qualifiers = JsonDictField('Non conflicting Qualifiers', compute='_compute_unique_qualifiers', store=True, help="Non conflicting qualifiers in common needed to link error content.") + analogous_ids = fields.One2many('runbot.build.error', compute='_compute_analogous_ids', string="Analogous Errors", help="Analogous Errors based on unique qualifiers") + analogous_content_ids= fields.One2many('runbot.build.error.content', compute='_compute_analogous_content_ids', string="Analogous Error Contents", help="Analogous Error contents based on unique qualifiers") + # Build error related data build_error_link_ids = fields.Many2many('runbot.build.error.link', compute=_compute_related_error_content_ids('build_error_link_ids'), search=_search_related_error_content_ids('build_error_link_ids')) unique_build_error_link_ids = fields.Many2many('runbot.build.error.link', compute='_compute_unique_build_error_link_ids') @@ -149,6 +156,84 @@ class BuildError(models.Model): for record in self: record.random = any(error.random for error in record.error_content_ids) + @api.depends('error_content_ids.qualifiers') + def _compute_common_qualifiers(self): + for record in self: + qualifiers = defaultdict(set) + key_count = defaultdict(int) + for content in record.error_content_ids: + for key, value in content.qualifiers.dict.items(): + qualifiers[key].add(value) + key_count[key] += 1 + record.common_qualifiers = {k: v.pop() for k, v in qualifiers.items() if len(v) == 1 and key_count[k] == len(record.error_content_ids)} + + @api.depends('error_content_ids.qualifiers') + def _compute_unique_qualifiers(self): + for record in self: + qualifiers = defaultdict(set) + key_count = defaultdict(int) + for content in record.error_content_ids: + for key, value in content.qualifiers.dict.items(): + qualifiers[key].add(value) + key_count[key] += 1 + record.unique_qualifiers = {k: v.pop() for k, v in qualifiers.items() if len(v) == 1} + + @api.depends('common_qualifiers') + def _compute_similar_ids(self): + for record in self: + if record.common_qualifiers: + query = SQL( + r"""SELECT id FROM runbot_build_error WHERE id != %s AND common_qualifiers @> %s""", + record.id, + json.dumps(record.common_qualifiers.dict), + ) + self.env.cr.execute(query) + record.similar_ids = self.env['runbot.build.error'].browse([rec[0] for rec in self.env.cr.fetchall()]) + else: + record.similar_ids = False + + @api.depends('common_qualifiers') + def _compute_similar_content_ids(self): + for record in self: + if record.common_qualifiers: + query = SQL( + r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""", + record.id, + json.dumps(record.common_qualifiers.dict), + ) + self.env.cr.execute(query) + record.similar_content_ids = self.env['runbot.build.error.content'].browse([rec[0] for rec in self.env.cr.fetchall()]) + else: + record.similar_content_ids = False + + @api.depends('common_qualifiers') + def _compute_analogous_ids(self): + for record in self: + if record.common_qualifiers: + query = SQL( + r"""SELECT id FROM runbot_build_error WHERE id != %s AND unique_qualifiers @> %s""", + record.id, + json.dumps(record.unique_qualifiers.dict), + ) + self.env.cr.execute(query) + record.analogous_ids = self.env['runbot.build.error'].browse([rec[0] for rec in self.env.cr.fetchall()]) + else: + record.analogous_ids = False + + @api.depends('common_qualifiers') + def _compute_analogous_content_ids(self): + for record in self: + if record.common_qualifiers: + query = SQL( + r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""", + record.id, + json.dumps(record.unique_qualifiers.dict), + ) + self.env.cr.execute(query) + record.analogous_content_ids = self.env['runbot.build.error.content'].browse([rec[0] for rec in self.env.cr.fetchall()]) + else: + record.analogous_content_ids = False + @api.constrains('test_tags') def _check_test_tags(self): @@ -202,6 +287,8 @@ class BuildError(models.Model): if not error.team_id: error.team_id = previous_error.team_id previous_error.error_content_ids.write({'error_id': self}) + previous_error.common_qualifiers = dict() + previous_error.unique_qualifiers = dict() if not previous_error.test_tags: previous_error.message_post(body=Markup('Error merged into %s') % error._get_form_link()) previous_error.active = False @@ -245,6 +332,50 @@ class BuildError(models.Model): 'target': 'current', } + def action_view_similary_qualified(self): + return { + 'type': 'ir.actions.act_window', + 'views': [(False, 'list'), (False, 'form')], + 'res_model': 'runbot.build.error', + 'domain': [('id', 'in', [self.id] + self.similar_ids.ids)], + 'context': {'active_test': False}, + 'target': 'current', + 'name': 'Similary Qualified Errors' + } + + def action_view_similary_qualified_contents(self): + return { + 'type': 'ir.actions.act_window', + 'views': [(False, 'list'), (False, 'form')], + 'res_model': 'runbot.build.error.content', + 'domain': [('id', 'in', self.similar_content_ids.ids)], + 'context': {'active_test': False}, + 'target': 'current', + 'name': 'Similary Qualified Contents' + } + + def action_view_analogous_qualified(self): + return { + 'type': 'ir.actions.act_window', + 'views': [(False, 'list'), (False, 'form')], + 'res_model': 'runbot.build.error', + 'domain': [('id', 'in', [self.id] + self.analogous_ids.ids)], + 'context': {'active_test': False}, + 'target': 'current', + 'name': 'Similary Qualified Errors' + } + + def action_view_analogous_qualified_contents(self): + return { + 'type': 'ir.actions.act_window', + 'views': [(False, 'list'), (False, 'form')], + 'res_model': 'runbot.build.error.content', + 'domain': [('id', 'in', self.analogous_content_ids.ids)], + 'context': {'active_test': False}, + 'target': 'current', + 'name': 'Similary Qualified Contents' + } + def action_assign(self): teams = None repos = None @@ -731,7 +862,8 @@ for error_content in self: result = False if content and self.regex: result = re.search(self.regex, content, flags=re.MULTILINE) - return result.groupdict() if result else {} + # filtering empty values to allow non mandatory named groups + return {k:v for k,v in result.groupdict().items() if v} if result else {} class QualifyErrorTest(models.Model): diff --git a/runbot/tests/test_build_error.py b/runbot/tests/test_build_error.py index e165e716..6f55fd9f 100644 --- a/runbot/tests/test_build_error.py +++ b/runbot/tests/test_build_error.py @@ -529,6 +529,61 @@ class TestBuildError(RunbotCase): self.assertEqual([], self.BuildError._disabling_tags(build_fixing)) self.assertEqual(['-bar'], self.BuildError._disabling_tags(build_random)) + def test_build_error_qualifers(self): + error_contents = self.BuildErrorContent.create( + [ + { + "content": "Tour foobar_tour failed at step click_here in mode admin", + }, + { + "content": "Tour foobar_tour failed at step click_here in mode demo", + }, + { + "content": "Tour foobar_tour failed -> click_here", + }, + { + "content": "Tour foobar_tour failed", + }, + ] + ) + self.assertEqual(len(error_contents), 4) + self.assertEqual(len(self.BuildError.search([('error_content_ids', 'in', error_contents.ids)])), 4) + + self.env['runbot.error.qualify.regex'].create({ + "regex": r"Tour (?P\w+) failed( at step (?P\w+) in mode (?P\w+))?" + }) + error_contents._qualify() + + expected_common_qualifiers = {'tour_name': 'foobar_tour', 'tour_step': 'click_here'} + self.assertEqual(error_contents[0].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'admin'}) + self.assertEqual(error_contents[1].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'demo'}) + self.assertEqual(error_contents[2].qualifiers.dict, {'tour_name': 'foobar_tour'}) + + self.env['runbot.error.qualify.regex'].create({ + "regex": r"Tour (?P\w+) failed -> (?P\w+)" + }) + + error_contents._qualify() + self.assertEqual(error_contents[0].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'admin'}) + self.assertEqual(error_contents[1].qualifiers.dict, {**expected_common_qualifiers, 'tour_mode': 'demo'}) + self.assertEqual(error_contents[2].qualifiers.dict, expected_common_qualifiers) + + # now let's say that we merge admin and demo errors + main_error = error_contents[0].error_id + self.assertEqual(len(main_error.error_content_ids), 1) + main_error._merge(error_contents[1].error_id) + self.assertEqual(len(main_error.error_content_ids), 2) + + self.assertEqual(main_error.common_qualifiers.dict, expected_common_qualifiers) + self.env.flush_all() + self.assertEqual(len(main_error.similar_ids), 1) + self.assertEqual(main_error.similar_ids[0], error_contents[2].error_id) + + # let's merge all errors and verify the qualifiers + main_error._merge(error_contents.error_id) + self.assertEqual(main_error.common_qualifiers.dict, {'tour_name': 'foobar_tour'}) + self.assertEqual(main_error.unique_qualifiers.dict, {'tour_name': 'foobar_tour', 'tour_step': 'click_here'}) + def test_build_error_team_wildcards(self): website_team = self.RunbotTeam.create({ 'name': 'website_test', diff --git a/runbot/views/build_error_views.xml b/runbot/views/build_error_views.xml index 105a0659..a0e66684 100644 --- a/runbot/views/build_error_views.xml +++ b/runbot/views/build_error_views.xml @@ -53,7 +53,7 @@ - + @@ -82,6 +82,124 @@ + +
+ +
+ + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + +