[IMP] runbot: add build_error model

With this commit, a new RunbotBuilError model is added in order to
classify and manage errors that appears during runbot builds.  This is
an helper to find undeterministic bugs in Odoo builds.  Build logs can
be parsed on demand, during the parsing, the logs are cleaned with some
regexes stored on the RunbotErrorRegex model.  A hash is computed on the
cleaned log, if a build error already exists with the same fingerprint,
the build is appended on the build error.

Errors can also be manually linked together with a parent/children
relation in case of a related error log. e.g. the error message is
different in two different branches but the bug is the same.

Also, a new build_url field is added to the runbot_build in order to
access the build web page from the backend.
This commit is contained in:
Christophe Monniez 2019-06-27 17:32:00 +02:00 committed by Xavier-Do
parent 02d2cc4528
commit 0da30a9f60
11 changed files with 403 additions and 3 deletions

View File

@ -17,6 +17,7 @@
'views/branch_views.xml',
'views/build_views.xml',
'views/host_views.xml',
'views/build_error_views.xml',
'views/config_views.xml',
'views/res_config_settings_views.xml',
'templates/frontend.xml',
@ -27,5 +28,8 @@
'templates/badge.xml',
'templates/branch.xml',
'data/runbot_build_config_data.xml',
'data/build_parse.xml',
'data/runbot_error_regex_data.xml',
'data/error_link.xml',
],
}

View File

@ -0,0 +1,12 @@
<odoo>
<record model="ir.actions.server" id="action_parse_build_logs">
<field name="name">Parse build logs</field>
<field name="model_id" ref="runbot.model_runbot_build" />
<field name="binding_model_id" ref="runbot.model_runbot_build" />
<field name="type">ir.actions.server</field>
<field name="state">code</field>
<field name="code">
records._parse_logs()
</field>
</record>
</odoo>

View File

@ -0,0 +1,22 @@
<odoo>
<record model="ir.actions.server" id="action_link_build_errors">
<field name="name">Link build errors</field>
<field name="model_id" ref="runbot.model_runbot_build_error" />
<field name="binding_model_id" ref="runbot.model_runbot_build_error" />
<field name="type">ir.actions.server</field>
<field name="state">code</field>
<field name="code">
records.link_errors()
</field>
</record>
<record model="ir.actions.server" id="action_clean_build_errors">
<field name="name">Re-clean build errors</field>
<field name="model_id" ref="runbot.model_runbot_build_error" />
<field name="binding_model_id" ref="runbot.model_runbot_build_error" />
<field name="type">ir.actions.server</field>
<field name="state">code</field>
<field name="code">
records.clean_content()
</field>
</record>
</odoo>

View File

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<odoo>
<data noupdate="1">
<record id="runbot_error_regex_clean_numbers" model="runbot.error.regex">
<field name="regex">, line \d+,</field>
<field name="re_type">cleaning</field>
</record>
<record id="runbot_error_regex_filter_failures" model="runbot.error.regex">
<field name="regex">Module .+: \d+ failures, \d+ errors</field>
<field name="re_type">filter</field>
</record>
<record id="runbot_error_regex_filter_failed" model="runbot.error.regex">
<field name="regex">At least one test failed when loading the modules.</field>
<field name="re_type">filter</field>
</record>
</data>
</odoo>

View File

@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
from . import repo, branch, build, event, build_dependency, build_config, ir_cron, host
from . import repo, branch, build, event, build_dependency, build_config, ir_cron, host, build_error
from . import res_config_settings

View File

@ -104,6 +104,9 @@ class runbot_build(models.Model):
],
default='soft',
string='Source export path mode')
build_url = fields.Char('Build url', compute='_compute_build_url', store=False)
build_error_ids = fields.Many2many('runbot.build.error', 'runbot_build_error_ids_runbot_build_rel', string='Errors')
@api.depends('config_id')
def _compute_log_list(self): # storing this field because it will be access trhoug repo viewn and keep track of the list at create
for build in self:
@ -342,6 +345,10 @@ class runbot_build(models.Model):
else:
build.domain = "%s:%s" % (domain, build.port)
def _compute_build_url(self):
for build in self:
build.build_url = "/runbot/build/%s" % build.id
@api.depends('job_start', 'job_end', 'duplicate_id.job_time')
def _compute_job_time(self):
"""Return the time taken by the tests"""
@ -988,6 +995,14 @@ class runbot_build(models.Model):
return '3'
return ''
def _parse_logs(self):
""" Parse build logs to classify errors """
BuildError = self.env['runbot.build.error']
# only parse logs from builds in error and not already scanned
builds_to_scan = self.search([('id', 'in', self.ids), ('local_result', '=', 'ko'), ('build_error_ids', '=', False)])
ir_logs = self.env['ir.logging'].search([('level', '=', 'ERROR'), ('type', '=', 'server'), ('build_id', 'in', builds_to_scan.ids)])
BuildError._parse_logs(ir_logs)
def read_file(self, file, mode='r'):
file_path = self._path(file)
try:

View File

@ -0,0 +1,148 @@
# -*- coding: utf-8 -*-
import hashlib
import logging
import re
from collections import defaultdict
from odoo import models, fields, api
_logger = logging.getLogger(__name__)
class RunbotBuildError(models.Model):
_name = "runbot.build.error"
_inherit = "mail.thread"
_rec_name = "id"
content = fields.Text('Error message', required=True)
cleaned_content = fields.Text('Cleaned error message')
summary = fields.Char('Content summary', compute='_compute_summary', store=False)
module_name = fields.Char('Module name') # name in ir_logging
function = fields.Char('Function name') # func name in ir logging
fingerprint = fields.Char('Error fingerprint', index=True)
random = fields.Boolean('underterministic error', track_visibility='onchange')
responsible = fields.Many2one('res.users', 'Assigned fixer', track_visibility='onchange')
fixing_commit = fields.Char('Fixing commit', track_visibility='onchange')
build_ids = fields.Many2many('runbot.build', 'runbot_build_error_ids_runbot_build_rel', string='Affected builds')
branch_ids = fields.Many2many('runbot.branch', compute='_compute_branch_ids')
repo_ids = fields.Many2many('runbot.repo', compute='_compute_repo_ids')
active = fields.Boolean('Error is not fixed', default=True, track_visibility='onchange')
tag_ids = fields.Many2many('runbot.build.error.tag', string='Tags')
build_count = fields.Integer(compute='_compute_build_counts', string='Nb seen', stored=True)
parent_id = fields.Many2one('runbot.build.error', 'Linked to')
child_ids = fields.One2many('runbot.build.error', 'parent_id', string='Child Errors')
Children_build_ids = fields.Many2many(related='child_ids.build_ids', string='Children builds')
@api.model
def create(self, vals):
cleaners = self.env['runbot.error.regex'].search([('re_type', '=', 'cleaning')])
content = vals.get('content')
cleaned_content = cleaners.r_sub('%', content)
vals.update({'cleaned_content': cleaned_content,
'fingerprint': self._digest(cleaned_content)
})
return super().create(vals)
@api.depends('build_ids')
def _compute_build_counts(self):
for build_error in self:
build_error.build_count = len(build_error.build_ids) + len(build_error.Children_build_ids)
@api.depends('build_ids')
def _compute_branch_ids(self):
for build_error in self:
build_error.branch_ids = build_error.mapped('build_ids.branch_id')
@api.depends('build_ids')
def _compute_repo_ids(self):
for build_error in self:
build_error.repo_ids = build_error.mapped('build_ids.repo_id')
@api.depends('content')
def _compute_summary(self):
for build_error in self:
build_error.summary = build_error.content[:50]
@api.model
def _digest(self, s):
"""
return a hash 256 digest of the string s
"""
return hashlib.sha256(s.encode()).hexdigest()
@api.model
def _parse_logs(self, ir_logs):
regexes = self.env['runbot.error.regex'].search([])
search_regs = regexes.filtered(lambda r: r.re_type == 'filter')
cleaning_regs = regexes.filtered(lambda r: r.re_type == 'cleaning')
hash_dict = defaultdict(list)
for log in ir_logs:
if search_regs.r_search(log.message):
continue
fingerprint = self._digest(cleaning_regs.r_sub('%', log.message))
hash_dict[fingerprint].append(log)
# add build ids to already detected errors
for build_error in self.env['runbot.build.error'].search([('fingerprint', 'in', list(hash_dict.keys()))]):
for build in {rec.build_id for rec in hash_dict[build_error.fingerprint]}:
build.build_error_ids += build_error
del hash_dict[build_error.fingerprint]
# create an error for the remaining entries
for fingerprint, logs in hash_dict.items():
self.env['runbot.build.error'].create({
'content': logs[0].message,
'module_name': logs[0].name,
'function': logs[0].func,
'build_ids': [(6, False, [r.build_id.id for r in logs])],
})
def link_errors(self):
""" Link errors with the first one of the recordset
choosing parent in error with responsible, random bug and finally fisrt seen
"""
if len(self) < 2:
return
build_errors = self.search([('id', 'in', self.ids)], order='responsible asc, random desc, id asc')
build_errors[1:].write({'parent_id': build_errors[0].id})
def clean_content(self):
cleaning_regs = self.env['runbot.error.regex'].search([('re_type', '=', 'cleaning')])
for build_error in self:
build_error.cleaned_content = cleaning_regs.r_sub('%', build_error.content)
class RunbotBuildErrorTag(models.Model):
_name = "runbot.build.error.tag"
name = fields.Char('Tag')
error_ids = fields.Many2many('runbot.build.error', string='Errors')
class RunbotErrorRegex(models.Model):
_name = "runbot.error.regex"
_inherit = "mail.thread"
_rec_name = 'id'
_order = 'sequence, id'
regex = fields.Char('Regular expression')
re_type = fields.Selection([('filter', 'Filter out'), ('cleaning', 'Cleaning')], string="Regex type")
sequence = fields.Integer('Sequence', default=100)
def r_sub(self, replace, s):
""" replaces patterns from the recordset by replace in the given string """
for c in self:
s = re.sub(c.regex, '%', s)
return s
def r_search(self, s):
""" Return True if one of the regex is found in s """
for filter in self:
if re.search(filter.regex, s):
return True
return False

View File

@ -18,5 +18,13 @@ access_runbot_build_config_manager,runbot_build_config_manager,runbot.model_runb
access_runbot_build_config_step_order_user,runbot_build_config_step_order_user,runbot.model_runbot_build_config_step_order,group_user,1,0,0,0
access_runbot_build_config_step_order_manager,runbot_build_config_step_order_manager,runbot.model_runbot_build_config_step_order,runbot.group_build_config_user,1,1,1,1
access_runbot_build_error_user,runbot_build_error_user,runbot.model_runbot_build_error,group_user,1,0,0,0
access_runbot_build_error_manager,runbot_build_error_manager,runbot.model_runbot_build_error,runbot.group_runbot_admin,1,1,1,1
access_runbot_build_error_tag_user,runbot_build_error_tag_user,runbot.model_runbot_build_error_tag,group_user,1,0,0,0
access_runbot_build_error_tag_manager,runbot_build_error_tag_manager,runbot.model_runbot_build_error_tag,runbot.group_runbot_admin,1,1,1,1
access_runbot_error_regex_user,runbot_error_regex_user,runbot.model_runbot_error_regex,group_user,1,0,0,0
access_runbot_error_regex_manager,runbot_error_regex_manager,runbot.model_runbot_error_regex,runbot.group_runbot_admin,1,1,1,1
access_runbot_host_user,runbot_host_user,runbot.model_runbot_host,group_user,1,0,0,0
access_runbot_host_manager,runbot_host_manager,runbot.model_runbot_host,runbot.group_runbot_admin,1,1,1,1
access_runbot_host_manager,runbot_host_manager,runbot.model_runbot_host,runbot.group_runbot_admin,1,1,1,1
1 id name model_id:id group_id:id perm_read perm_write perm_create perm_unlink
18 access_runbot_host_manager access_runbot_build_error_manager runbot_host_manager runbot_build_error_manager runbot.model_runbot_host runbot.model_runbot_build_error runbot.group_runbot_admin 1 1 1 1
19 access_runbot_build_error_tag_user runbot_build_error_tag_user runbot.model_runbot_build_error_tag group_user 1 0 0 0
20 access_runbot_build_error_tag_manager runbot_build_error_tag_manager runbot.model_runbot_build_error_tag runbot.group_runbot_admin 1 1 1 1
21 access_runbot_error_regex_user runbot_error_regex_user runbot.model_runbot_error_regex group_user 1 0 0 0
22 access_runbot_error_regex_manager runbot_error_regex_manager runbot.model_runbot_error_regex runbot.group_runbot_admin 1 1 1 1
23 access_runbot_host_user runbot_host_user runbot.model_runbot_host group_user 1 0 0 0
24 access_runbot_host_manager runbot_host_manager runbot.model_runbot_host runbot.group_runbot_admin 1 1 1 1
25
26
27
28
29
30

View File

@ -99,7 +99,7 @@
<li><a t-attf-href="https://{{repo.base}}/compare/{{br['branch'].branch_name}}">Compare <i class="fa fa-github"/></a></li>
<!-- TODO branch.pull from -->
<li class="divider"></li>
<li class="disabled"><a href="#">Port: <t t-esc="bu.real_build.port"/></a></li>
<li groups="runbot.group_runbot_admin"><a t-attf-href="/web/#id={{bu['id']}}&amp;view_type=form&amp;model=runbot.build" target="new">View in backend</a></li>
</ul>
</div>
</div>

View File

@ -0,0 +1,172 @@
<odoo>
<data>
<record id="build_error_form" model="ir.ui.view">
<field name="name">runbot.build.error.form</field>
<field name="model">runbot.build.error</field>
<field name="arch" type="xml">
<form>
<header>
</header>
<sheet>
<group name="build_error_group">
<field name="fingerprint"/>
<field name="content"/>
<field name="cleaned_content" groups="base.group_no_one"/>
<field name="module_name"/>
<field name="function"/>
<field name="random"/>
<field name="responsible"/>
<field name="fixing_commit"/>
<field name="active"/>
<field name="parent_id" />
<field name="branch_ids" widget="many2many_tags"/>
<field name="repo_ids" widget="many2many_tags"/>
<field name="tag_ids" widget="many2many_tags"/>
</group>
<field name="build_ids" nolabel="1" widget="many2many" options="{'not_delete': True, 'no_create': True}">
<tree>
<field name="create_date"/>
<field name="id"/>
<field name="repo_id"/>
<field name="branch_id"/>
<field name="dest"/>
<field name="build_url" widget="url" readonly="1" text="View build"/>
</tree>
</field>
<label for="Children_build_ids" string="Linked errors builds"/>
<field name="Children_build_ids" widget="many2many" options="{'not_delete': True, 'no_create': True}">
<tree>
<field name="create_date"/>
<field name="id"/>
<field name="repo_id"/>
<field name="branch_id"/>
<field name="dest"/>
<field name="build_url" widget="url" readonly="1" text="View build"/>
</tree>
</field>
</sheet>
<div class="oe_chatter">
<field name="message_follower_ids" widget="mail_followers"/>
<field name="message_ids" widget="mail_thread"/>
</div>
</form>
</field>
</record>
<record id="build_error_view_tree" model="ir.ui.view">
<field name="name">runbot.build.error.tree</field>
<field name="model">runbot.build.error</field>
<field name="arch" type="xml">
<tree string="Errors">
<field name="module_name"/>
<field name="summary"/>
<field name="random"/>
<field name="build_count"/>
<field name="responsible"/>
<field name="fixing_commit"/>
</tree>
</field>
</record>
<record id="build_error_search_view" model="ir.ui.view">
<field name="name">runbot.build.error.log.filter</field>
<field name="model">runbot.build.error</field>
<field name="arch" type="xml">
<search string="Search errors">
<field name="content"/>
<field name="module_name"/>
<field name="function"/>
<field name="responsible"/>
<field name="fixing_commit"/>
<filter string="No Parent" name="no_parent_error" domain="[('parent_id', '=', False)]"/>
<separator/>
<filter string="Undeterministic" name="random_error" domain="[('random', '=', True)]"/>
<separator/>
<filter string="Fixed" name="fixed_errors" domain="[('active', '=', False)]"/>
<filter string="Not Fixed" name="not_fixed_errors" domain="[('active', '=', True)]"/>
<separator/>
<filter string="Not Asigned" name="not_assigned_errors" domain="[('responsible', '=', False)]"/>
</search>
</field>
</record>
<record id="open_view_build_error_tree" model="ir.actions.act_window">
<field name="name">Build errors</field>
<field name="res_model">runbot.build.error</field>
<field name="view_mode">tree,form</field>
<field name="context">{'search_default_no_parent_error': True, 'search_default_random_error': True}</field>
</record>
<record id="build_error_regex_form" model="ir.ui.view">
<field name="name">runbot.error.regex.form</field>
<field name="model">runbot.error.regex</field>
<field name="arch" type="xml">
<form>
<sheet>
<group name="build_regex_group">
<field name="regex"/>
<field name="re_type"/>
</group>
</sheet>
<div class="oe_chatter">
<field name="message_follower_ids" widget="mail_followers"/>
<field name="message_ids" widget="mail_thread"/>
</div>
</form>
</field>
</record>
<record id="build_error_regex_tree" model="ir.ui.view">
<field name="name">runbot.error.regex.tree</field>
<field name="model">runbot.error.regex</field>
<field name="arch" type="xml">
<tree string="Errors Regexes">
<field name="sequence" widget="handle"/>
<field name="regex"/>
<field name="re_type"/>
</tree>
</field>
</record>
<record id="runbot_regex_search_view" model="ir.ui.view">
<field name="name">runbot.error.regex.filter</field>
<field name="model">runbot.error.regex</field>
<field name="arch" type="xml">
<search string="Search regex">
<field name="regex"/>
<filter string="Filtering regex's" name="filtering_regex" domain="[(['re_type', '=', 'filter'])]"/>
<filter string="Cleaning regex's" name="clening_regex" domain="[(['re_type', '=', 'cleaning'])]"/>
</search>
</field>
</record>
<record id="open_view_error_regex" model="ir.actions.act_window">
<field name="name">Errors regex</field>
<field name="res_model">runbot.error.regex</field>
<field name="view_mode">tree,form</field>
</record>
<menuitem
name="Manage errors"
id="runbot_menu_manage_errors"
parent="runbot_menu_root"
sequence="40"
/>
<menuitem
name="Build errors"
id="runbot_menu_build_error_tree"
parent="runbot_menu_manage_errors"
sequence="10"
action="open_view_build_error_tree"
/>
<menuitem
name="Error regex"
id="runbot_menu_error_regex_tree"
parent="runbot_menu_manage_errors"
sequence="20"
action="open_view_error_regex"
/>
</data>
</odoo>

View File

@ -37,6 +37,7 @@
<field name="config_id" groups="base.group_no_one"/>
<field name="orphan_result" readonly="1"/>
<field name="hidden" groups="base.group_no_one"/>
<field name="build_url" widget="url" readonly="1"/>
</group>
</sheet>
</form>
@ -77,6 +78,7 @@
<field name="model">runbot.build</field>
<field name="arch" type="xml">
<search string="Search builds">
<field name="id"/>
<field name="branch_id"/>
<field name="name"/>
<field name="global_state"/>