mirror of
https://github.com/odoo/runbot.git
synced 2025-03-15 23:45:44 +07:00
[IMP] runbot: add a build stat model
When a build is done, various numerical informations could be extracted from log files. e.g.: global query count or tests query count ... The extraction regular expression could be hard-coded in a custom step but there is no place holder where to store the retrieved information. In order to compare results, we need to store it. With this commit, a new model `runbot.build.stat` is used to store key/values pair linked to a build/config_step. That way, extracted values can be stored. Also, another `runbot.build.stat.regex` is used to store regular expressions that can be used to grep log files and extract values. The regular expression must contain a named group like this: `(?P<value>.+)` The text catched by this group MUST be castable into a float. Optionally, another named group can be used in the regular expresion like this: `(?P<key>.+)` This `key` group will then be used to augment the key name in the database. Example: Consider a log line like this one: `odoo.addons.website_blog.tests.test_ui tested in 10.35s` A regular expression like this, named `test_duration`: `odoo.addons.(?P<key>.+) tested in (?P<value>\d+\.\d+)s` Should store the following key:value: `{ 'key': 'test_duration.website_blog.tests.test_ui', 'value': 10.35 }` A `generic` boolean field is present on the build.stat.regex object, meaning that when no regex are linked to a make_stats config step, then, all the generic regex will be applied. A wizard is added to help the creation the regular expressions, allowing to test if they work against a user provided example. A _make_stats method is added to the ConfigStep model which is called during the _schedule of a build, just before calling the next step. The regex search is only apllied in steps that have the `make_stats` boolean field set to true. Also, the build branch have to be flagged `make_stats` too or the build can have a key `make_stats` in its config_data field. The `make_stats` field on the branch is a compute stored field. That way, sticky branches are automaticaly set `make_stats' true. Finally, an SQL view is used to facilitate the stats visualisation.
This commit is contained in:
parent
03d7f871be
commit
360e31ade4
@ -6,7 +6,7 @@
|
||||
'author': "Odoo SA",
|
||||
'website': "http://runbot.odoo.com",
|
||||
'category': 'Website',
|
||||
'version': '4.9',
|
||||
'version': '4.10',
|
||||
'depends': ['website', 'base'],
|
||||
'data': [
|
||||
'security/runbot_security.xml',
|
||||
@ -21,7 +21,9 @@
|
||||
'views/error_log_views.xml',
|
||||
'views/config_views.xml',
|
||||
'views/res_config_settings_views.xml',
|
||||
'views/stat_views.xml',
|
||||
'wizards/mutli_build_wizard_views.xml',
|
||||
'wizards/stat_regex_wizard_views.xml',
|
||||
'templates/frontend.xml',
|
||||
'templates/build.xml',
|
||||
'templates/assets.xml',
|
||||
|
@ -1,4 +1,14 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from . import repo, branch, build, event, build_dependency, build_config, ir_cron, host, build_error
|
||||
from . import repo
|
||||
from . import branch
|
||||
from . import build
|
||||
from . import event
|
||||
from . import build_dependency
|
||||
from . import build_config
|
||||
from . import ir_cron
|
||||
from . import host
|
||||
from . import build_error
|
||||
from . import build_stat
|
||||
from . import build_stat_regex
|
||||
from . import res_config_settings
|
||||
|
@ -41,6 +41,8 @@ class runbot_branch(models.Model):
|
||||
branch_config_id = fields.Many2one('runbot.build.config', 'Branch Config')
|
||||
config_id = fields.Many2one('runbot.build.config', 'Run Config', compute='_compute_config_id', inverse='_inverse_config_id')
|
||||
|
||||
make_stats = fields.Boolean('Extract stats from logs', compute='_compute_make_stats', store=True)
|
||||
|
||||
@api.depends('sticky', 'defined_sticky', 'target_branch_name', 'name')
|
||||
# won't be recompute if a new branch is marked as sticky or sticky is removed, but should be ok if not stored
|
||||
def _compute_closest_sticky(self):
|
||||
@ -107,6 +109,11 @@ class runbot_branch(models.Model):
|
||||
for branch in self:
|
||||
branch.pull_branch_name = branch.pull_head_name.split(':')[-1] if branch.pull_head_name else branch.branch_name
|
||||
|
||||
@api.depends('sticky')
|
||||
def _compute_make_stats(self):
|
||||
for branch in self:
|
||||
branch.make_stats = branch.sticky
|
||||
|
||||
@api.depends('name')
|
||||
def _get_branch_infos(self, pull_info=None):
|
||||
"""compute branch_name, branch_url, pull_head_name and target_branch_name based on name"""
|
||||
|
@ -65,7 +65,7 @@ class runbot_build(models.Model):
|
||||
repo_id = fields.Many2one(related='branch_id.repo_id', readonly=True, store=True)
|
||||
name = fields.Char('Revno', required=True)
|
||||
description = fields.Char('Description', help='Informative description')
|
||||
md_description = fields.Char(compute='_compute_md_description', String='MD Parsed Description', help='Informative description mardown parsed')
|
||||
md_description = fields.Char(compute='_compute_md_description', String='MD Parsed Description', help='Informative description markdown parsed')
|
||||
host = fields.Char('Host')
|
||||
port = fields.Integer('Port')
|
||||
dest = fields.Char(compute='_compute_dest', type='char', string='Dest', readonly=1, store=True)
|
||||
@ -80,6 +80,7 @@ class runbot_build(models.Model):
|
||||
log_ids = fields.One2many('ir.logging', 'build_id', string='Logs')
|
||||
error_log_ids = fields.One2many('ir.logging', 'build_id', domain=[('level', 'in', ['WARNING', 'ERROR', 'CRITICAL'])], string='Error Logs')
|
||||
config_data = JsonDictField('Config Data')
|
||||
stat_ids = fields.One2many('runbot.build.stat', 'build_id', strings='Statistics values')
|
||||
|
||||
# state machine
|
||||
|
||||
@ -738,6 +739,9 @@ class runbot_build(models.Model):
|
||||
|
||||
build_values.update(results)
|
||||
|
||||
# compute statistics before starting next job
|
||||
build.active_step._make_stats(build)
|
||||
|
||||
build.active_step.log_end(build)
|
||||
|
||||
build_values.update(build._next_job_values()) # find next active_step or set to done
|
||||
|
@ -102,6 +102,8 @@ class ConfigStep(models.Model):
|
||||
step_order_ids = fields.One2many('runbot.build.config.step.order', 'step_id')
|
||||
group = fields.Many2one('runbot.build.config', 'Configuration group', help="Group of config's and config steps")
|
||||
group_name = fields.Char('Group name', related='group.name')
|
||||
make_stats = fields.Boolean('Make stats', default=False)
|
||||
build_stat_regex_ids = fields.Many2many('runbot.build.stat.regex', string='Stats Regexes')
|
||||
# install_odoo
|
||||
create_db = fields.Boolean('Create Db', default=True, track_visibility='onchange') # future
|
||||
custom_db_name = fields.Char('Custom Db Name', track_visibility='onchange') # future
|
||||
@ -571,6 +573,23 @@ class ConfigStep(models.Model):
|
||||
build_values['local_result'] = build._get_worst_result([build.local_result, local_result])
|
||||
return build_values
|
||||
|
||||
def _make_stats(self, build):
|
||||
build._log('make_stats', 'Getting stats from log file')
|
||||
log_path = build._path('logs', '%s.txt' % self.name)
|
||||
if not os.path.exists(log_path):
|
||||
build._log('make_stats', 'Log **%s.txt** file not found' % self.name, level='INFO', log_type='markdown')
|
||||
return
|
||||
if (build.branch_id.make_stats or build.config_data.get('make_stats')) and self.make_stats:
|
||||
try:
|
||||
regex_ids = self.build_stat_regex_ids
|
||||
if not regex_ids:
|
||||
regex_ids = regex_ids.search([('generic', '=', True)])
|
||||
key_values = regex_ids._find_in_file(log_path)
|
||||
self.env['runbot.build.stat']._write_key_values(build, self, key_values)
|
||||
except Exception as e:
|
||||
message = '**An error occured while computing statistics of %s:**\n`%s`' % (build.job, str(e).replace('\\n', '\n').replace("\\'", "'"))
|
||||
build._log('make_stats', message, level='INFO', log_type='markdown')
|
||||
|
||||
def _step_state(self):
|
||||
self.ensure_one()
|
||||
if self.job_type == 'run_odoo' or (self.job_type == 'python' and self.running_job):
|
||||
|
98
runbot/models/build_stat.py
Normal file
98
runbot/models/build_stat.py
Normal file
@ -0,0 +1,98 @@
|
||||
import logging
|
||||
|
||||
from odoo import models, fields, api, tools
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RunbotBuildStat(models.Model):
|
||||
_name = "runbot.build.stat"
|
||||
_description = "Statistics"
|
||||
_sql_constraints = [
|
||||
(
|
||||
"build_config_key_unique",
|
||||
"unique (build_id, config_step_id,key)",
|
||||
"Build stats must be unique for the same build step",
|
||||
)
|
||||
]
|
||||
|
||||
build_id = fields.Many2one("runbot.build", "Build", index=True, ondelete="cascade")
|
||||
config_step_id = fields.Many2one(
|
||||
"runbot.build.config.step", "Step", index=True, ondelete="cascade"
|
||||
)
|
||||
key = fields.Char("key", index=True)
|
||||
value = fields.Float("Value")
|
||||
|
||||
@api.model
|
||||
def _write_key_values(self, build, config_step, key_values):
|
||||
if not key_values:
|
||||
return self
|
||||
build_stats = [
|
||||
{
|
||||
"config_step_id": config_step.id,
|
||||
"build_id": build.id,
|
||||
"key": k,
|
||||
"value": v,
|
||||
}
|
||||
for k, v in key_values.items()
|
||||
]
|
||||
return self.create(build_stats)
|
||||
|
||||
|
||||
class RunbotBuildStatSql(models.Model):
|
||||
|
||||
_name = "runbot.build.stat.sql"
|
||||
_description = "Build stat sql view"
|
||||
_auto = False
|
||||
|
||||
id = fields.Many2one("runbot.build.stat", readonly=True)
|
||||
key = fields.Char("Key", readonly=True)
|
||||
value = fields.Float("Value", readonly=True)
|
||||
config_step_id = fields.Many2one(
|
||||
"runbot.build.config.step", string="Config Step", readonly=True
|
||||
)
|
||||
config_step_name = fields.Char(String="Config Step name", readonly=True)
|
||||
build_id = fields.Many2one("runbot.build", string="Build", readonly=True)
|
||||
build_config_id = fields.Many2one("runbot.build.config", string="Config", readonly=True)
|
||||
build_name = fields.Char(String="Build name", readonly=True)
|
||||
build_parent_path = fields.Char('Build Parent path')
|
||||
host = fields.Char(string="Host", readonly=True)
|
||||
branch_id = fields.Many2one("runbot.branch", string="Branch", readonly=True)
|
||||
branch_name = fields.Char(string="Branch name", readonly=True)
|
||||
branch_sticky = fields.Boolean(string="Sticky", readonly=True)
|
||||
repo_id = fields.Many2one("runbot.repo", string="Repo", readonly=True)
|
||||
repo_name = fields.Char(string="Repo name", readonly=True)
|
||||
|
||||
def init(self):
|
||||
""" Create SQL view for build stat """
|
||||
tools.drop_view_if_exists(self._cr, "runbot_build_stat_sql")
|
||||
self._cr.execute(
|
||||
""" CREATE VIEW runbot_build_stat_sql AS (
|
||||
SELECT
|
||||
stat.id AS id,
|
||||
stat.key AS key,
|
||||
stat.value AS value,
|
||||
step.id AS config_step_id,
|
||||
step.name AS config_step_name,
|
||||
bu.id AS build_id,
|
||||
bu.config_id AS build_config_id,
|
||||
bu.parent_path AS build_parent_path,
|
||||
bu.name AS build_name,
|
||||
bu.host AS build_host,
|
||||
br.id AS branch_id,
|
||||
br.branch_name AS branch_name,
|
||||
br.sticky AS branch_sticky,
|
||||
repo.id AS repo_id,
|
||||
repo.name AS repo_name
|
||||
FROM
|
||||
runbot_build_stat AS stat
|
||||
JOIN
|
||||
runbot_build_config_step step ON stat.config_step_id = step.id
|
||||
JOIN
|
||||
runbot_build bu ON stat.build_id = bu.id
|
||||
JOIN
|
||||
runbot_branch br ON br.id = bu.branch_id
|
||||
JOIN
|
||||
runbot_repo repo ON br.repo_id = repo.id
|
||||
)"""
|
||||
)
|
72
runbot/models/build_stat_regex.py
Normal file
72
runbot/models/build_stat_regex.py
Normal file
@ -0,0 +1,72 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
from odoo import models, fields, api
|
||||
from odoo.exceptions import ValidationError
|
||||
|
||||
VALUE_PATTERN = r"\(\?P\<value\>.+\)" # used to verify value group pattern
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RunbotBuildStatRegex(models.Model):
|
||||
""" A regular expression to extract a float/int value from a log file
|
||||
The regulare should contain a named group like '(?P<value>.+)'.
|
||||
The result will be a key/value like {name: value}
|
||||
A second named group '(?P<key>.+)' can bu used to augment the key name
|
||||
like {name.key_result: value}
|
||||
A 'generic' regex will be used when no regex are defined on a make_stat
|
||||
step.
|
||||
"""
|
||||
|
||||
_name = "runbot.build.stat.regex"
|
||||
_description = "Statistics regex"
|
||||
|
||||
name = fields.Char("Key Name")
|
||||
regex = fields.Char("Regular Expression")
|
||||
description = fields.Char("Description")
|
||||
generic = fields.Boolean('Generic', help='Executed when no regex on the step', default=True)
|
||||
config_step_ids = fields.Many2many('runbot.build.config.step', string='Config Steps')
|
||||
|
||||
@api.constrains("name", "regex")
|
||||
def _check_regex(self):
|
||||
for rec in self:
|
||||
try:
|
||||
r = re.compile(rec.regex)
|
||||
except re.error as e:
|
||||
raise ValidationError("Unable to compile regular expression: %s" % e)
|
||||
# verify that a named group exist in the pattern
|
||||
if not re.search(VALUE_PATTERN, r.pattern):
|
||||
raise ValidationError(
|
||||
"The regular expresion should contain the name group pattern 'value' e.g: '(?P<value>.+)'"
|
||||
)
|
||||
|
||||
def _find_in_file(self, file_path):
|
||||
""" Search file regexes and write stats
|
||||
returns a dict of key:values
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
return {}
|
||||
key_values = {}
|
||||
with open(file_path, "r") as log_file:
|
||||
data = log_file.read()
|
||||
for build_stat_regex in self:
|
||||
for match in re.finditer(build_stat_regex.regex, data):
|
||||
group_dict = match.groupdict()
|
||||
try:
|
||||
value = float(group_dict.get("value"))
|
||||
except ValueError:
|
||||
_logger.warning(
|
||||
'The matched value (%s) of "%s" cannot be converted into float'
|
||||
% (group_dict.get("value"), build_stat_regex.regex)
|
||||
)
|
||||
continue
|
||||
key = (
|
||||
"%s.%s" % (build_stat_regex.name, group_dict["key"])
|
||||
if "key" in group_dict
|
||||
else build_stat_regex.name
|
||||
)
|
||||
key_values[key] = value
|
||||
return key_values
|
@ -33,4 +33,13 @@ access_runbot_error_log_user,runbot_error_log_user,runbot.model_runbot_error_log
|
||||
access_runbot_error_log_manager,runbot_error_log_manager,runbot.model_runbot_error_log,runbot.group_runbot_admin,1,1,1,1
|
||||
|
||||
access_runbot_repo_hooktime,runbot_repo_hooktime,runbot.model_runbot_repo_hooktime,group_user,1,0,0,0
|
||||
access_runbot_repo_reftime,runbot_repo_reftime,runbot.model_runbot_repo_reftime,group_user,1,0,0,0
|
||||
access_runbot_repo_reftime,runbot_repo_reftime,runbot.model_runbot_repo_reftime,group_user,1,0,0,0
|
||||
|
||||
access_runbot_build_stat_user,runbot_build_stat_user,runbot.model_runbot_build_stat,group_user,1,0,0,0
|
||||
access_runbot_build_stat_admin,runbot_build_stat_admin,runbot.model_runbot_build_stat,runbot.group_runbot_admin,1,1,1,1
|
||||
|
||||
access_runbot_build_stat_sql_user,runbot_build_stat_sql_user,runbot.model_runbot_build_stat_sql,group_user,1,0,0,0
|
||||
access_runbot_build_stat_sql_admin,runbot_build_stat_sql_admin,runbot.model_runbot_build_stat_sql,runbot.group_runbot_admin,1,0,0,0
|
||||
|
||||
access_runbot_build_stat_regex_user,access_runbot_build_stat_regex_user,model_runbot_build_stat_regex,runbot.group_user,1,0,0,0
|
||||
access_runbot_build_stat_regex_admin,access_runbot_build_stat_regex_admin,model_runbot_build_stat_regex,runbot.group_runbot_admin,1,1,1,1
|
||||
|
|
@ -9,3 +9,4 @@ from . import test_cron
|
||||
from . import test_build_config_step
|
||||
from . import test_event
|
||||
from . import test_command
|
||||
from . import test_build_stat
|
||||
|
135
runbot/tests/test_build_stat.py
Normal file
135
runbot/tests/test_build_stat.py
Normal file
@ -0,0 +1,135 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from psycopg2 import IntegrityError
|
||||
from unittest.mock import patch, mock_open
|
||||
from odoo.exceptions import ValidationError
|
||||
from odoo.tools import mute_logger
|
||||
from .common import RunbotCase
|
||||
|
||||
|
||||
class TestBuildStatRegex(RunbotCase):
|
||||
def setUp(self):
|
||||
super(TestBuildStatRegex, self).setUp()
|
||||
self.StatRegex = self.env["runbot.build.stat.regex"]
|
||||
self.ConfigStep = self.env["runbot.build.config.step"]
|
||||
self.BuildStat = self.env["runbot.build.stat"]
|
||||
|
||||
self.repo = self.Repo.create(
|
||||
{
|
||||
"name": "bla@example.com:foo/bar",
|
||||
"server_files": "server.py",
|
||||
"addons_paths": "addons,core/addons",
|
||||
}
|
||||
)
|
||||
self.branch = self.Branch.create(
|
||||
{"repo_id": self.repo.id, "name": "refs/heads/master"}
|
||||
)
|
||||
|
||||
self.Build = self.env["runbot.build"]
|
||||
|
||||
self.build = self.create_build(
|
||||
{
|
||||
"branch_id": self.branch.id,
|
||||
"name": "d0d0caca0000ffffffffffffffffffffffffffff",
|
||||
"port": "1234",
|
||||
"config_data": {"make_stats": True},
|
||||
}
|
||||
)
|
||||
|
||||
self.config_step = self.env["runbot.build.config.step"].create(
|
||||
{
|
||||
"name": "a_nice_step",
|
||||
"job_type": "install_odoo",
|
||||
"make_stats": True,
|
||||
"build_stat_regex_ids": [(0, 0, {"name": "query_count", "regex": r"odoo.addons.(?P<key>.+) tested in .+, (?P<value>\d+) queries", "generic": False})]
|
||||
}
|
||||
)
|
||||
|
||||
def test_build_stat_regex_validation(self):
|
||||
|
||||
# test that a regex without a named key 'value' raises a ValidationError
|
||||
with self.assertRaises(ValidationError):
|
||||
self.StatRegex.create(
|
||||
{"name": "query_count", "regex": "All post-tested in .+s, .+ queries"}
|
||||
)
|
||||
|
||||
def test_build_stat_regex_find_in_file(self):
|
||||
|
||||
file_content = """foo bar
|
||||
2020-03-02 22:06:58,391 17 INFO xxx odoo.modules.module: odoo.addons.website_blog.tests.test_ui tested in 10.35s, 2501 queries
|
||||
some garbage
|
||||
2020-03-02 22:07:14,340 17 INFO xxx odoo.modules.module: odoo.addons.website_event.tests.test_ui tested in 9.26s, 2435 queries
|
||||
nothing to see here
|
||||
"""
|
||||
self.start_patcher(
|
||||
"isdir", "odoo.addons.runbot.models.build_stat_regex.os.path.exists", True
|
||||
)
|
||||
with patch("builtins.open", mock_open(read_data=file_content)):
|
||||
self.config_step._make_stats(self.build)
|
||||
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_blog.tests.test_ui'), ('value', '=', 2501.0)]), 1)
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_event.tests.test_ui'), ('value', '=', 2435.0)]), 1)
|
||||
|
||||
# Check unicity
|
||||
with self.assertRaises(IntegrityError):
|
||||
with mute_logger("odoo.sql_db"):
|
||||
with self.cr.savepoint(): # needed to continue tests
|
||||
self.env["runbot.build.stat"]._write_key_values(
|
||||
self.build, self.config_step, {'query_count.website_event.tests.test_ui': 2435}
|
||||
)
|
||||
|
||||
# minimal test for RunbotBuildStatSql model
|
||||
self.assertEqual(self.env['runbot.build.stat.sql'].search_count([('build_id', '=', self.build.id)]), 2)
|
||||
|
||||
def test_build_stat_regex_generic(self):
|
||||
""" test that regex are not used when generic is False and that _make_stats use all genreic regex if there are no regex on step """
|
||||
file_content = """foo bar
|
||||
odoo.addons.foobar tested in 2s, 25 queries
|
||||
useless 10
|
||||
chocolate 15
|
||||
"""
|
||||
|
||||
self.config_step.build_stat_regex_ids = False
|
||||
|
||||
# this one is not generic and thus should not be used
|
||||
self.StatRegex.create({"name": "useless_count", "regex": r"(?P<key>useless) (?P<value>\d+)", "generic": False})
|
||||
|
||||
# this is one is the only one that should be used
|
||||
self.StatRegex.create({"name": "chocolate_count", "regex": r"(?P<key>chocolate) (?P<value>\d+)"})
|
||||
|
||||
self.start_patcher(
|
||||
"isdir", "odoo.addons.runbot.models.build_stat_regex.os.path.exists", True
|
||||
)
|
||||
with patch("builtins.open", mock_open(read_data=file_content)):
|
||||
self.config_step._make_stats(self.build)
|
||||
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.foobar'), ('value', '=', 25.0)]), 0)
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'useless_count.useless'), ('value', '=', 10.0)]), 0)
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'chocolate_count.chocolate'), ('value', '=', 15.0)]), 1)
|
||||
|
||||
def test_build_stat_regex_find_in_file_perf(self):
|
||||
|
||||
noise_lines = """2020-03-17 13:26:15,472 2376 INFO runbottest odoo.modules.loading: loading runbot/views/build_views.xml
|
||||
2020-03-10 22:58:34,472 17 INFO 1709329-master-9938b2-all_no_autotag werkzeug: 127.0.0.1 - - [10/Mar/2020 22:58:34] "POST /mail/read_followers HTTP/1.1" 200 - 13 0.004 0.009
|
||||
2020-03-10 22:58:30,137 17 INFO ? werkzeug: 127.0.0.1 - - [10/Mar/2020 22:58:30] "GET /website/static/src/xml/website.editor.xml HTTP/1.1" 200 - - - -
|
||||
"""
|
||||
|
||||
match_lines = [
|
||||
"2020-03-02 22:06:58,391 17 INFO xxx odoo.modules.module: odoo.addons.website_blog.tests.test_ui tested in 10.35s, 2501 queries",
|
||||
"2020-03-02 22:07:14,340 17 INFO xxx odoo.modules.module: odoo.addons.website_event.tests.test_ui tested in 9.26s, 2435 queries"
|
||||
]
|
||||
|
||||
# generate a 13 MiB log file with two potential matches
|
||||
log_data = ""
|
||||
for l in match_lines:
|
||||
log_data += noise_lines * 10000
|
||||
log_data += l
|
||||
log_data += noise_lines * 10000
|
||||
|
||||
self.start_patcher(
|
||||
"isdir", "odoo.addons.runbot.models.build_stat_regex.os.path.exists", True
|
||||
)
|
||||
with patch("builtins.open", mock_open(read_data=log_data)):
|
||||
self.config_step._make_stats(self.build)
|
||||
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_blog.tests.test_ui'), ('value', '=', 2501.0)]), 1)
|
||||
self.assertEqual(self.BuildStat.search_count([('key', '=', 'query_count.website_event.tests.test_ui'), ('value', '=', 2435.0)]), 1)
|
@ -52,6 +52,17 @@
|
||||
<field name="gc_delay"/>
|
||||
</group>
|
||||
</sheet>
|
||||
<sheet>
|
||||
<group>
|
||||
<field name="stat_ids">
|
||||
<tree>
|
||||
<field name="config_step_id"/>
|
||||
<field name="key"/>
|
||||
<field name="value"/>
|
||||
</tree>
|
||||
</field>
|
||||
</group>
|
||||
</sheet>
|
||||
</form>
|
||||
</field>
|
||||
</record>
|
||||
|
@ -45,10 +45,20 @@
|
||||
<group string="General settings">
|
||||
<field name="name"/>
|
||||
<field name="job_type"/>
|
||||
<field name="make_stats"/>
|
||||
<field name="protected" groups="base.group_no_one"/>
|
||||
<field name="default_sequence" groups="base.group_no_one"/>
|
||||
<field name="group" groups="base.group_no_one"/>
|
||||
</group>
|
||||
<group string="Stats regexes" attrs="{'invisible': [('make_stats', '=', False)]}">
|
||||
<field name="build_stat_regex_ids">
|
||||
<tree string="Regexes" editable="bottom">
|
||||
<field name="name"/>
|
||||
<field name="regex"/>
|
||||
<field name="description"/>
|
||||
</tree>
|
||||
</field>
|
||||
</group>
|
||||
<group string="Python settings" attrs="{'invisible': [('job_type', 'not in', ('python'))]}">
|
||||
<field name="python_code" widget="ace" options="{'mode': 'python'}"/>
|
||||
<field name="python_result_code" widget="ace" options="{'mode': 'python'}"/>
|
||||
|
85
runbot/views/stat_views.xml
Normal file
85
runbot/views/stat_views.xml
Normal file
@ -0,0 +1,85 @@
|
||||
<odoo>
|
||||
<data>
|
||||
|
||||
<record id="view_stat_sql_tree" model="ir.ui.view">
|
||||
<field name="name">runbot.stat.sql.tree</field>
|
||||
<field name="model">runbot.build.stat.sql</field>
|
||||
<field name="arch" type="xml">
|
||||
<tree string="Statistics">
|
||||
<field name="config_step_id"/>
|
||||
<field name="build_id"/>
|
||||
<field name="build_name"/>
|
||||
<field name="repo_name"/>
|
||||
<field name="branch_name"/>
|
||||
<field name="key"/>
|
||||
<field name="value"/>
|
||||
</tree>
|
||||
</field>
|
||||
</record>
|
||||
|
||||
<record id="open_view_stat_sql_tree" model="ir.actions.act_window">
|
||||
<field name="name">Statistics</field>
|
||||
<field name="res_model">runbot.build.stat.sql</field>
|
||||
<field name="view_mode">tree,graph,pivot</field>
|
||||
</record>
|
||||
|
||||
<menuitem
|
||||
name="Build Statistics"
|
||||
id="runbot_menu_stat_root"
|
||||
parent="runbot_menu_root"
|
||||
sequence="100"
|
||||
/>
|
||||
|
||||
<menuitem
|
||||
name="Build Statistics"
|
||||
id="runbot_menu_stat_tree"
|
||||
parent="runbot_menu_stat_root"
|
||||
sequence="10"
|
||||
action="open_view_stat_sql_tree"
|
||||
/>
|
||||
|
||||
<record id="build_stat_regex_form" model="ir.ui.view">
|
||||
<field name="name">runbot.build.stat.regex.form</field>
|
||||
<field name="model">runbot.build.stat.regex</field>
|
||||
<field name="arch" type="xml">
|
||||
<form>
|
||||
<sheet>
|
||||
<group name="stat_regex_group">
|
||||
<field name="name"/>
|
||||
<field name="regex" placeholder="odoo.addons.(?P<key>.+) tested in .+, (?P<value>\d+) queries"/>
|
||||
<field name="generic"/>
|
||||
<field name="description"/>
|
||||
</group>
|
||||
</sheet>
|
||||
</form>
|
||||
</field>
|
||||
</record>
|
||||
|
||||
<record id="build_stat_regex_tree" model="ir.ui.view">
|
||||
<field name="name">runbot.build.stat.regex.tree</field>
|
||||
<field name="model">runbot.build.stat.regex</field>
|
||||
<field name="arch" type="xml">
|
||||
<tree string="Statistics Regexes">
|
||||
<field name="name"/>
|
||||
<field name="generic"/>
|
||||
<field name="description"/>
|
||||
<field name="regex"/>
|
||||
</tree>
|
||||
</field>
|
||||
</record>
|
||||
|
||||
<record id="open_view_stat_regex_tree" model="ir.actions.act_window">
|
||||
<field name="name">Stat regex</field>
|
||||
<field name="res_model">runbot.build.stat.regex</field>
|
||||
<field name="view_mode">tree,form</field>
|
||||
</record>
|
||||
|
||||
<menuitem
|
||||
name="Stats Regexes"
|
||||
id="runbot_menu_manage_stat_regexes"
|
||||
parent="runbot_menu_stat_root"
|
||||
sequence="20"
|
||||
action="open_view_stat_regex_tree"
|
||||
/>
|
||||
</data>
|
||||
</odoo>
|
@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from . import multi_build_wizard
|
||||
from . import stat_regex_wizard
|
||||
|
71
runbot/wizards/stat_regex_wizard.py
Normal file
71
runbot/wizards/stat_regex_wizard.py
Normal file
@ -0,0 +1,71 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import re
|
||||
|
||||
from odoo import fields, models, api
|
||||
from odoo.exceptions import ValidationError
|
||||
from odoo.addons.runbot.models.build_stat_regex import VALUE_PATTERN
|
||||
|
||||
|
||||
class StatRegexWizard(models.TransientModel):
|
||||
_name = 'runbot.build.stat.regex.wizard'
|
||||
_description = "Stat Regex Wizard"
|
||||
|
||||
name = fields.Char("Key Name")
|
||||
regex = fields.Char("Regular Expression")
|
||||
description = fields.Char("Description")
|
||||
generic = fields.Boolean('Generic', help='Executed when no regex on the step', default=True)
|
||||
test_text = fields.Text("Test text")
|
||||
key = fields.Char("Key")
|
||||
value = fields.Float("Value")
|
||||
message = fields.Char("Wizard message")
|
||||
|
||||
def _validate_regex(self):
|
||||
try:
|
||||
regex = re.compile(self.regex)
|
||||
except re.error as e:
|
||||
raise ValidationError("Unable to compile regular expression: %s" % e)
|
||||
if not re.search(VALUE_PATTERN, regex.pattern):
|
||||
raise ValidationError(
|
||||
"The regular expresion should contain the name group pattern 'value' e.g: '(?P<value>.+)'"
|
||||
)
|
||||
|
||||
@api.onchange('regex', 'test_text')
|
||||
def _onchange_regex(self):
|
||||
key = ''
|
||||
value = False
|
||||
self.message = ''
|
||||
if self.regex and self.test_text:
|
||||
self._validate_regex()
|
||||
match = re.search(self.regex, self.test_text)
|
||||
if match:
|
||||
group_dict = match.groupdict()
|
||||
try:
|
||||
value = float(group_dict.get("value"))
|
||||
except ValueError:
|
||||
raise ValidationError('The matched value (%s) of "%s" cannot be converted into float' % (group_dict.get("value"), self.regex))
|
||||
key = (
|
||||
"%s.%s" % (self.name, group_dict["key"])
|
||||
if "key" in group_dict
|
||||
else self.name
|
||||
)
|
||||
else:
|
||||
self.message = 'No match !'
|
||||
self.key = key
|
||||
self.value = value
|
||||
|
||||
def save(self):
|
||||
if self.regex and self.test_text:
|
||||
self._validate_regex()
|
||||
stat_regex = self.env['runbot.build.stat.regex'].create({
|
||||
'name': self.name,
|
||||
'regex': self.regex,
|
||||
'description': self.description,
|
||||
'generic': self.generic,
|
||||
})
|
||||
return {
|
||||
'name': 'Stat regex',
|
||||
'type': 'ir.actions.act_window',
|
||||
'res_model': 'runbot.build.stat.regex',
|
||||
'view_mode': 'form',
|
||||
'res_id': stat_regex.id
|
||||
}
|
46
runbot/wizards/stat_regex_wizard_views.xml
Normal file
46
runbot/wizards/stat_regex_wizard_views.xml
Normal file
@ -0,0 +1,46 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<odoo>
|
||||
<data>
|
||||
<record model="ir.ui.view" id="runbot_stat_regex_wizard_form">
|
||||
<field name="name">runbot_stat_regex_wizard</field>
|
||||
<field name="model">runbot.build.stat.regex.wizard</field>
|
||||
<field name="arch" type="xml">
|
||||
<form string="Regex">
|
||||
<group>
|
||||
<field name="name"/>
|
||||
<field name="regex"/>
|
||||
<field name="description"/>
|
||||
<field name="generic"/>
|
||||
<field name="test_text"/>
|
||||
</group>
|
||||
<group>
|
||||
<field name="key" readonly="1"/>
|
||||
<field name="value" readonly="1"/>
|
||||
<field name="message" readonly="1"/>
|
||||
</group>
|
||||
<footer>
|
||||
<button name="save" string="Save" type="object" class="btn-primary"/>
|
||||
<button string="Cancel" special="cancel" class="btn-default"/>
|
||||
</footer>
|
||||
</form>
|
||||
</field>
|
||||
</record>
|
||||
|
||||
<record model="ir.actions.act_window" id="runbot_stat_regex_wizard_action">
|
||||
<field name="name">Generate Stat Regex</field>
|
||||
<field name="type">ir.actions.act_window</field>
|
||||
<field name="res_model">runbot.build.stat.regex.wizard</field>
|
||||
<field name="view_mode">form</field>
|
||||
<field name="view_id" ref="runbot_stat_regex_wizard_form"/>
|
||||
<field name="target">new</field>
|
||||
</record>
|
||||
|
||||
<menuitem
|
||||
name="Stat Regex Wizard"
|
||||
id="runbot_menu_stat_regex_wizard"
|
||||
parent="runbot_menu_stat_root"
|
||||
sequence="150"
|
||||
action="runbot_stat_regex_wizard_action"
|
||||
/>
|
||||
</data>
|
||||
</odoo>
|
Loading…
Reference in New Issue
Block a user