From 1790f14f7c00f1a367eb29dd873e5829d1f3824a Mon Sep 17 00:00:00 2001 From: Xavier-Do Date: Fri, 14 Mar 2025 17:22:09 +0100 Subject: [PATCH] wip --- runbot/__manifest__.py | 2 + runbot/controllers/__init__.py | 1 + runbot/controllers/errors.py | 11 ++++ runbot/models/__init__.py | 1 + runbot/models/build_error.py | 51 +++++++++++------ runbot/models/build_error_merge.py | 71 ++++++++++++++++++++++++ runbot/security/ir.model.access.csv | 5 ++ runbot/templates/error_merge.xml | 37 ++++++++++++ runbot/tests/__init__.py | 1 + runbot/tests/test_build_error_merge.py | 9 +++ runbot/views/build_error_merge_views.xml | 44 +++++++++++++++ runbot/views/menus.xml | 4 +- 12 files changed, 220 insertions(+), 17 deletions(-) create mode 100644 runbot/controllers/errors.py create mode 100644 runbot/models/build_error_merge.py create mode 100644 runbot/templates/error_merge.xml create mode 100644 runbot/tests/test_build_error_merge.py create mode 100644 runbot/views/build_error_merge_views.xml diff --git a/runbot/__manifest__.py b/runbot/__manifest__.py index 5085df56..a5f3cbba 100644 --- a/runbot/__manifest__.py +++ b/runbot/__manifest__.py @@ -32,6 +32,7 @@ 'templates/bundle.xml', 'templates/commit.xml', 'templates/dashboard.xml', + 'templates/error_merge.xml', 'templates/frontend.xml', 'templates/git.xml', 'templates/nginx.xml', @@ -40,6 +41,7 @@ 'views/branch_views.xml', 'views/build_error_link_views.xml', 'views/build_error_views.xml', + 'views/build_error_merge_views.xml', 'views/build_views.xml', 'views/bundle_views.xml', 'views/codeowner_views.xml', diff --git a/runbot/controllers/__init__.py b/runbot/controllers/__init__.py index 96d149ab..23066c11 100644 --- a/runbot/controllers/__init__.py +++ b/runbot/controllers/__init__.py @@ -3,3 +3,4 @@ from . import frontend from . import hook from . import badge +from . import errors \ No newline at end of file diff --git a/runbot/controllers/errors.py b/runbot/controllers/errors.py new file mode 100644 index 00000000..dfd3441a --- /dev/null +++ b/runbot/controllers/errors.py @@ -0,0 +1,11 @@ +from odoo.http import Controller, Response, request, route + + +class ErrorControlle(Controller): + + @route('/runbot/error/merge/result/', type='http', auth='public', website=True) + def error_filter_result(self, filter_id=None, **kwargs): + merger = request.env['runbot.build.error.merge'].browse(int(filter_id)) + if not merger: + return Response('Error merge not found', status=404) + return request.render('runbot.error_merge_result', {'merger': merger, 'results': merger._get_matching_groups()}) \ No newline at end of file diff --git a/runbot/models/__init__.py b/runbot/models/__init__.py index dbd376be..fbb0f087 100644 --- a/runbot/models/__init__.py +++ b/runbot/models/__init__.py @@ -6,6 +6,7 @@ from . import build from . import build_config from . import build_config_codeowner from . import build_error +from . import build_error_merge from . import bundle from . import codeowner from . import commit diff --git a/runbot/models/build_error.py b/runbot/models/build_error.py index c2eccda4..ce9fdf1d 100644 --- a/runbot/models/build_error.py +++ b/runbot/models/build_error.py @@ -232,8 +232,8 @@ class BuildError(models.Model): for record in self: if record.common_qualifiers: query = SQL( - r"""SELECT id FROM runbot_build_error WHERE id != %s AND common_qualifiers @> %s""", - record.id, + r"""SELECT id FROM runbot_build_error WHERE id not in %s AND common_qualifiers @> %s""", + tuple(record.ids), json.dumps(record.common_qualifiers.dict), ) self.env.cr.execute(query) @@ -246,8 +246,8 @@ class BuildError(models.Model): for record in self: if record.common_qualifiers: query = SQL( - r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""", - record.id, + r"""SELECT id FROM runbot_build_error_content WHERE error_id not in %s AND qualifiers @> %s""", + tuple(record.ids), json.dumps(record.common_qualifiers.dict), ) self.env.cr.execute(query) @@ -260,8 +260,8 @@ class BuildError(models.Model): for record in self: if record.common_qualifiers: query = SQL( - r"""SELECT id FROM runbot_build_error WHERE id != %s AND unique_qualifiers @> %s""", - record.id, + r"""SELECT id FROM runbot_build_error WHERE id not in %s AND unique_qualifiers @> %s""", + tuple(record.ids), json.dumps(record.unique_qualifiers.dict), ) self.env.cr.execute(query) @@ -274,8 +274,8 @@ class BuildError(models.Model): for record in self: if record.common_qualifiers: query = SQL( - r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""", - record.id, + r"""SELECT id FROM runbot_build_error_content WHERE error_id not in %s AND qualifiers @> %s""", + tuple(record.ids), json.dumps(record.unique_qualifiers.dict), ) self.env.cr.execute(query) @@ -537,6 +537,9 @@ class BuildError(models.Model): existing_errors_contents = self.env['runbot.build.error.content'].search([('fingerprint', 'in', list(hash_dict.keys())), ('error_id.active', '=', True)]) existing_fingerprints = {error.fingerprint: error for error in existing_errors_contents} build_error_contents |= existing_errors_contents + + # TODO + error_content_merge = self.env['runbot.build.error.merge'].search([('auto_merge', '=', True), ('active', '=', True)]) # create an error for the remaining entries for fingerprint, logs in hash_dict.items(): if fingerprint in existing_fingerprints: @@ -546,13 +549,24 @@ class BuildError(models.Model): error.metadata = logs[0].metadata continue - new_build_error_content = self.env['runbot.build.error.content'].create({ + vals = { + 'error_id': None, 'content': logs[0].message, 'module_name': logs[0].name.removeprefix('odoo.').removeprefix('addons.'), 'file_path': logs[0].path, 'function': logs[0].func, 'metadata': logs[0].metadata, - }) + 'canonical_tag': logs[0].metadata.get('test', {}).get('canonical_tag') + } + self._qualify(vals) # populate vals with qualifiers + similar_domain = error_content_merge._get_similar_domain(vals) + error_candidates = self.env['runbot.build.error.content'].search(similar_domain) + if error_candidates: + vals['error_id'] = error_candidates[0].error_id.id + + new_build_error_content = self.env['runbot.build.error.content'].create() + + build_error_contents |= new_build_error_content existing_fingerprints[fingerprint] = new_build_error_content @@ -595,6 +609,7 @@ class BuildErrorContent(models.Model): _inherit = ('mail.thread', 'mail.activity.mixin', 'runbot.build.error.seen.mixin') _rec_name = "id" + active = fields.Boolean('Active', related='error_id.active') error_id = fields.Many2one('runbot.build.error', 'Linked to', index=True, required=True) error_display_id = fields.Integer(compute='_compute_error_display_id', string="Error id") content = fields.Text('Error message', required=True) @@ -614,7 +629,7 @@ class BuildErrorContent(models.Model): version_ids = fields.One2many('runbot.version', compute='_compute_version_ids', string='Versions', search='_search_version') trigger_ids = fields.Many2many('runbot.trigger', compute='_compute_trigger_ids', string='Triggers', search='_search_trigger_ids') tag_ids = fields.Many2many('runbot.build.error.tag', string='Tags') - qualifiers = JsonDictField('Qualifiers', index=True) + qualifiers = JsonDictField('Qualifiers', index=True, store=True, compute="_compute_qualifiers") similar_ids = fields.One2many('runbot.build.error.content', compute='_compute_similar_ids') responsible = fields.Many2one(related='error_id.responsible') @@ -716,8 +731,8 @@ class BuildErrorContent(models.Model): for record in self: if record.qualifiers: query = SQL( - r"""SELECT id FROM runbot_build_error_content WHERE id != %s AND qualifiers @> %s AND qualifiers <@ %s""", - record.id, + r"""SELECT id FROM runbot_build_error_content WHERE id not in %s AND qualifiers @> %s AND qualifiers <@ %s""", + tuple(record.ids), json.dumps(record.qualifiers.dict), json.dumps(record.qualifiers.dict), ) @@ -726,6 +741,10 @@ class BuildErrorContent(models.Model): else: record.similar_ids = False + @api.depends('content', 'canonical_tag', 'module_name', 'file_path', 'function') + def _compute_qualifiers(self): + self._qualify() + @api.model def _digest(self, s): """ @@ -779,16 +798,16 @@ class BuildErrorContent(models.Model): domain = [('id', 'in', self.ids)] if self else [] return [r[1] for r in self._read_group(domain, ('fingerprint'), ('id:array_agg'), [('id:count', '>', 1)])] - def _qualify(self): + def _qualify(self, vals): qualify_regexes = self.env['runbot.error.qualify.regex'].search([]) - for record in self: + for record in self or vals: all_qualifiers = {} for qualify_regex in qualify_regexes: res = qualify_regex._qualify(record) if res: # res.update({'qualifier_id': qualify_regex.id}) Probably not a good idea all_qualifiers.update(res) - record.qualifiers = all_qualifiers + record['qualifiers'] = all_qualifiers #################### # Actions diff --git a/runbot/models/build_error_merge.py b/runbot/models/build_error_merge.py new file mode 100644 index 00000000..c148f1ea --- /dev/null +++ b/runbot/models/build_error_merge.py @@ -0,0 +1,71 @@ +from odoo import models, fields, api +from odoo.osv import expression + +class BuildErrorMerge(models.Model): + _name = 'runbot.build.error.merge' + _description = 'Error Merge patterns' + _inherit = ['mail.thread'] + + active = fields.Boolean('Active', default=True) + name = fields.Char('Name', required=True) + merge_filter_ids = fields.One2many('runbot.build.error.merge.filters', 'error_merge_id', 'Merge Lines') + description = fields.Char('Description', compute='_compute_description', store=True, tracking=True) + oneline_description = fields.Char('One Line Description', compute='_compute_description_online') + auto_merge = fields.Boolean('Auto Merge', default=False) + + def _get_read_group_params(self): + domain = [('active', '=', True)] + for filter in self.merge_filter_ids: + domain = expression.AND([domain, [(filter.field_name, '!=', False)]]) + groups = self.merge_filter_ids.mapped('field_name') + assert groups + + return ( + domain, + groups, + ) + + def _get_matching_groups(self): + domain, groups = self._get_read_group_params() + result = self.env['runbot.build.error.content']._read_group( + domain, + groups, + ['id:array_agg'], + [('error_id:count_distinct', '>', 1)] + ) + return result + + def _get_similar_domain(self, error_content): + result = [expression.FALSE_LEAF] + for record in self: + if all(error_content[f.field_name] for f in record.merge_filter_ids): + merge_domain = [(f.field_name, '==', error_content[f.field_name]) for f in record.merge_filter_ids] + result = expression.OR([result, merge_domain]) + return result + + def action_see_matches(self): + self.ensure_one() + return { + 'name': 'Error Candidates', + 'type': 'ir.actions.act_url', + 'url': f"/runbot/error/merge/result/{self.id}", + } + + @api.depends('merge_filter_ids.field_name') + def _compute_description(self): + for record in self: + record.description = '\n'.join(f.field_name for f in record.merge_filter_ids) + + @api.depends('description') + def _compute_description_online(self): + for record in self: + record.oneline_description = record.description.replace('\n', ', ') + + +class BuildErrorMergeFilter(models.Model): + _name = 'runbot.build.error.merge.filters' + _description = 'Error Merge patterns filters' + + field = fields.Many2one('ir.model.fields', 'Field', domain=[('model_id.model', '=', 'runbot.build.error.content')], required=True, ondelete='cascade') + field_name = fields.Char('Field Name', related='field.name', store=True, readonly=True) + error_merge_id = fields.Many2one('runbot.build.error.merge', 'Error Merge', required=True) \ No newline at end of file diff --git a/runbot/security/ir.model.access.csv b/runbot/security/ir.model.access.csv index cd54fa04..8766494e 100644 --- a/runbot/security/ir.model.access.csv +++ b/runbot/security/ir.model.access.csv @@ -151,4 +151,9 @@ access_runbot_build_stat_regex_wizard,access_runbot_build_stat_regex_wizard,mode access_runbot_host_message,access_runbot_host_message,runbot.model_runbot_host_message,runbot.group_runbot_admin,1,0,0,0 +access_runbot_build_error_merge,access_runbot_build_error_merge,runbot.model_runbot_build_error_merge,base.group_user,1,0,0,0 +access_runbot_build_error_merge_filters,access_runbot_build_error_merge_filters,runbot.model_runbot_build_error_merge_filters,base.group_user,1,0,0,0 +access_runbot_build_error_merge,access_runbot_build_error_merge,runbot.model_runbot_build_error_merge,runbot.group_runbot_admin,1,1,1,1 +access_runbot_build_error_merge_filters,access_runbot_build_error_merge_filters,runbot.model_runbot_build_error_merge_filters,runbot.group_runbot_admin,1,1,1,1 + diff --git a/runbot/templates/error_merge.xml b/runbot/templates/error_merge.xml new file mode 100644 index 00000000..4a493e23 --- /dev/null +++ b/runbot/templates/error_merge.xml @@ -0,0 +1,37 @@ + + + + + + diff --git a/runbot/tests/__init__.py b/runbot/tests/__init__.py index e4410f29..e6b953f9 100644 --- a/runbot/tests/__init__.py +++ b/runbot/tests/__init__.py @@ -1,5 +1,6 @@ from . import common from . import test_batch +from . import test_build_error_merge from . import test_repo from . import test_build_error from . import test_branch diff --git a/runbot/tests/test_build_error_merge.py b/runbot/tests/test_build_error_merge.py new file mode 100644 index 00000000..11c38409 --- /dev/null +++ b/runbot/tests/test_build_error_merge.py @@ -0,0 +1,9 @@ +from .common import RunbotCase + + + +class TestErrorMerge(self): + + def test_auto_merge(self): + + def test_inactive_no_auto_merge(self): diff --git a/runbot/views/build_error_merge_views.xml b/runbot/views/build_error_merge_views.xml new file mode 100644 index 00000000..faddb26d --- /dev/null +++ b/runbot/views/build_error_merge_views.xml @@ -0,0 +1,44 @@ + + + + runbot.build.error.merge.form + runbot.build.error.merge + +
+
+
+ + + + + + + + + + + + + + +
+ + + runbot.build.error.merge.list + runbot.build.error.merge + + + + + + + + + + Error merge + runbot.build.error.merge + list,form + + + diff --git a/runbot/views/menus.xml b/runbot/views/menus.xml index 19063a9a..eb41ae5a 100644 --- a/runbot/views/menus.xml +++ b/runbot/views/menus.xml @@ -38,6 +38,8 @@ + + @@ -59,7 +61,7 @@ - +