NextERP/runbot
5
0
Fork 0
mirror of https://github.com/odoo/runbot.git synced 2025-03-15 15:35:46 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

[IMP] runbot: replace jobs by build configs

Browse Source 
This commit aims to replace static jobs by fully configurable build config.

Each build has a config (custom or inherited from repo or branch).
Each config has a list of steps.
For now, a step can test/run odoo or create a new child build. A python job is
also available.

The mimic the previous behaviour of runbot, a default config is available with
three steps, an install of base, an install+test of all modules, and a last step
for run.

Multibuilds are replace by a config containing cretaion steps.
The created builds are not displayed in main views, but are available
on parent build log page. The result of a parent takes the result of
all children into account.

This new mechanics will help to create some custom behaviours for specifics
use cases, and latter help to parallelise work.
This commit is contained in:
Xavier-Do 2019-04-22 15:49:33 +02:00 committed by Christophe Monniez
parent bb35b1cc9d
commit 8ef6bcfde7
41 changed files with 1562 additions and 897 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
runbot/__init__.py
View File

@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
from . import croninterval
from . import controllers
from . import models
from . import common

3
runbot/__manifest__.py
View File

@ -12,9 +12,11 @@
'security/runbot_security.xml',
'security/ir.model.access.csv',
'security/ir.rule.csv',
'views/assets.xml',
'views/repo_views.xml',
'views/branch_views.xml',
'views/build_views.xml',
'views/config_views.xml',
'views/res_config_settings_views.xml',
'templates/frontend.xml',
'templates/build.xml',
@ -23,5 +25,6 @@
'templates/nginx.xml',
'templates/badge.xml',
'templates/branch.xml',
'data/runbot_build_config_data.xml',
],
}

2
runbot/container.py
View File

@ -71,7 +71,7 @@ def docker_run(run_cmd, log_path, build_dir, container_name, exposed_ports=None,
"""
_logger.debug('Docker run command: %s', run_cmd)
logs = open(log_path, 'w')
logs.write("Docker command:\n%s\n=================================================\n" % run_cmd.replace('&& ', '&&\n').replace('|| ','||\n\t'))
# create start script
docker_command = [
'docker', 'run', '--rm',

17
runbot/controllers/badge.py
View File

@ -19,13 +19,15 @@ class RunbotBadge(Controller):
domain = [('repo_id', '=', repo_id),
('branch_id.branch_name', '=', branch),
('branch_id.sticky', '=', True),
('state', 'in', ['testing', 'running', 'done']),
('result', 'not in', ['skipped', 'manually_killed']),
('hidden', '=', False),
('parent_id', '=', False),
('global_state', 'in', ['testing', 'running', 'done']),
('global_result', 'not in', ['skipped', 'manually_killed']),
]
last_update = '__last_update'
builds = request.env['runbot.build'].sudo().search_read(
domain, ['state', 'result', 'job_age', last_update],
domain, ['global_state', 'global_result', 'build_age', last_update],
order='id desc', limit=1)
if not builds:
@ -38,14 +40,14 @@ class RunbotBadge(Controller):
if etag == retag:
return werkzeug.wrappers.Response(status=304)
if build['state'] == 'testing':
state = 'testing'
if build['global_state'] in ('testing', 'waiting'):
state = build['global_state']
cache_factor = 1
else:
cache_factor = 2
if build['result'] == 'ok':
if build['global_result'] == 'ok':
state = 'success'
elif build['result'] == 'warn':
elif build['global_result'] == 'warn':
state = 'warning'
else:
state = 'failed'
@ -53,6 +55,7 @@ class RunbotBadge(Controller):
# from https://github.com/badges/shields/blob/master/colorscheme.json
color = {
'testing': "#dfb317",
'waiting': "#dfb317",
'success': "#4c1",
'failed': "#e05d44",
'warning': "#fe7d37",

95
runbot/controllers/frontend.py
View File

@ -6,43 +6,17 @@ from collections import OrderedDict
from odoo.addons.http_routing.models.ir_http import slug
from odoo.addons.website.controllers.main import QueryURL
from odoo.http import Controller, request, route
from ..common import uniq_list, flatten, s2human
from ..common import uniq_list, flatten, fqdn
from odoo.osv import expression
class Runbot(Controller):
def build_info(self, build):
real_build = build.duplicate_id if build.state == 'duplicate' else build
return {
'id': build.id,
'name': build.name,
'state': real_build.state,
'result': real_build.result,
'guess_result': real_build.guess_result,
'subject': build.subject,
'author': build.author,
'committer': build.committer,
'dest': build.dest,
'real_dest': real_build.dest,
'job_age': s2human(real_build.job_age),
'job_time': s2human(real_build.job_time),
'job': real_build.job,
'domain': real_build.domain,
'host': real_build.host,
'port': real_build.port,
'server_match': real_build.server_match,
'duplicate_of': build.duplicate_id if build.state == 'duplicate' else False,
'coverage': build.coverage or build.branch_id.coverage,
'revdep_build_ids': sorted(build.revdep_build_ids, key=lambda x: x.repo_id.name),
'build_type' : build.build_type,
'build_type_label': dict(build.fields_get('build_type', 'selection')['build_type']['selection']).get(build.build_type, build.build_type),
}
def _pending(self):
ICP = request.env['ir.config_parameter'].sudo().get_param
warn = int(ICP('runbot.pending.warning', 5))
crit = int(ICP('runbot.pending.critical', 12))
count = request.env['runbot.build'].search_count([('state', '=', 'pending')])
count = request.env['runbot.build'].search_count([('local_state', '=', 'pending')])
level = ['info', 'warning', 'danger'][int(count > warn) + int(count > crit)]
return count, level
@ -71,16 +45,17 @@ class Runbot(Controller):
build_ids = []
if repo:
filters = {key: kwargs.get(key, '1') for key in ['pending', 'testing', 'running', 'done', 'deathrow']}
# FIXME or removeme (filters are broken)
filters = {key: kwargs.get(key, '1') for key in ['waiting', 'pending', 'testing', 'running', 'done', 'deathrow']}
domain = [('repo_id', '=', repo.id)]
domain += [('state', '!=', key) for key, value in iter(filters.items()) if value == '0']
domain += [('global_state', '!=', key) for key, value in iter(filters.items()) if value == '0']
if search:
search_domain = []
for to_search in search.split("|"):
search_domain = ['|', '|', '|'] + search_domain
search_domain += [('dest', 'ilike', to_search), ('subject', 'ilike', to_search), ('branch_id.branch_name', 'ilike', to_search)]
domain += search_domain[1:]
domain = expression.AND([domain, [('hidden', '=', False)]]) # don't display children builds on repo view
build_ids = build_obj.search(domain, limit=100)
branch_ids, build_by_branch_ids = [], {}
@ -109,7 +84,7 @@ class Runbot(Controller):
FROM
runbot_branch br INNER JOIN runbot_build bu ON br.id=bu.branch_id
WHERE
br.id in %s
br.id in %s AND (bu.hidden = 'f' OR bu.hidden IS NULL)
GROUP BY br.id, bu.id
ORDER BY br.id, bu.id DESC
) AS br_bu
@ -129,16 +104,18 @@ class Runbot(Controller):
def branch_info(branch):
return {
'branch': branch,
'builds': [self.build_info(build_dict[build_id]) for build_id in build_by_branch_ids.get(branch.id) or []]
'fqdn': fqdn(),
'builds': [build_dict[build_id] for build_id in build_by_branch_ids.get(branch.id) or []]
}
context.update({
'branches': [branch_info(b) for b in branches],
'testing': build_obj.search_count([('repo_id', '=', repo.id), ('state', '=', 'testing')]),
'running': build_obj.search_count([('repo_id', '=', repo.id), ('state', '=', 'running')]),
'pending': build_obj.search_count([('repo_id', '=', repo.id), ('state', '=', 'pending')]),
'testing': build_obj.search_count([('repo_id', '=', repo.id), ('local_state', '=', 'testing')]),
'running': build_obj.search_count([('repo_id', '=', repo.id), ('local_state', '=', 'running')]),
'pending': build_obj.search_count([('repo_id', '=', repo.id), ('local_state', '=', 'pending')]),
'qu': QueryURL('/runbot/repo/' + slug(repo), search=search, refresh=refresh, **filters),
'filters': filters,
'fqdn': fqdn(),
})
# consider host gone if no build in last 100
@ -148,8 +125,8 @@ class Runbot(Controller):
if result['host']:
context['host_stats'].append({
'host': result['host'],
'testing': build_obj.search_count([('state', '=', 'testing'), ('host', '=', result['host'])]),
'running': build_obj.search_count([('state', '=', 'running'), ('host', '=', result['host'])]),
'testing': build_obj.search_count([('local_state', '=', 'testing'), ('host', '=', result['host'])]),
'running': build_obj.search_count([('local_state', '=', 'running'), ('host', '=', result['host'])]),
})
return request.render('runbot.repo', context)
@ -176,12 +153,10 @@ class Runbot(Controller):
if not build.exists():
return request.not_found()
real_build = build.duplicate_id if build.state == 'duplicate' else build
# other builds
build_ids = Build.search([('branch_id', '=', build.branch_id.id)], limit=100)
other_builds = Build.browse(build_ids)
domain = [('build_id', '=', real_build.id)]
domain = [('build_id', '=', build.real_build.id)]
log_type = request.params.get('type', '')
if log_type:
domain.append(('type', '=', log_type))
@ -194,7 +169,8 @@ class Runbot(Controller):
context = {
'repo': build.repo_id,
'build': self.build_info(build),
'build': build,
'fqdn': fqdn(),
'br': {'branch': build.branch_id},
'logs': Logging.sudo().browse(logging_ids).ids,
'other_builds': other_builds.ids,
@ -221,15 +197,15 @@ class Runbot(Controller):
if builds:
last_build = False
for build in builds:
if build.state == 'running' or (build.state == 'duplicate' and build.duplicate_id.state == 'running'):
last_build = build if build.state == 'running' else build.duplicate_id
if build.local_state == 'running' or (build.local_state == 'duplicate' and build.duplicate_id.local_state == 'running'):
last_build = build if build.local_state == 'running' else build.duplicate_id
break
if not last_build:
# Find the last build regardless the state to propose a rebuild
last_build = builds[0]
if last_build.state != 'running':
if last_build.local_state != 'running':
url = "/runbot/build/%s?ask_rebuild=1" % last_build.id
else:
url = build.branch_id._get_branch_quickconnect_url(last_build.domain, last_build.dest)[build.branch_id.id]
@ -277,21 +253,22 @@ class Runbot(Controller):
r.update({
'name': repo.name,
'base': repo.base,
'testing': count([('repo_id', '=', repo.id), ('state', '=', 'testing')]),
'running': count([('repo_id', '=', repo.id), ('state', '=', 'running')]),
'pending': count([('repo_id', '=', repo.id), ('state', '=', 'pending')]),
'testing': count([('repo_id', '=', repo.id), ('local_state', '=', 'testing')]),
'running': count([('repo_id', '=', repo.id), ('local_state', '=', 'running')]),
'pending': count([('repo_id', '=', repo.id), ('local_state', '=', 'pending')]),
})
b = r['branches'].setdefault(branch.id, {'name': branch.branch_name, 'builds': list()})
b['builds'].append(self.build_info(build))
b['builds'].append(build)
# consider host gone if no build in last 100
build_threshold = max(builds.ids or [0]) - 100
for result in RB.read_group([('id', '>', build_threshold)], ['host'], ['host']):
if result['host']:
qctx['host_stats'].append({
'fqdn': fqdn(),
'host': result['host'],
'testing': count([('state', '=', 'testing'), ('host', '=', result['host'])]),
'running': count([('state', '=', 'running'), ('host', '=', result['host'])]),
'testing': count([('local_state', '=', 'testing'), ('host', '=', result['host'])]),
'running': count([('local_state', '=', 'running'), ('host', '=', result['host'])]),
})
return request.render("runbot.sticky-dashboard", qctx)
@ -302,18 +279,17 @@ class Runbot(Controller):
query = """
SELECT split_part(r.name, ':', 2),
br.branch_name,
(array_agg(coalesce(du.result, bu.result) order by bu.id desc))[1]
(array_agg(bu.global_result order by bu.id desc))[1]
FROM runbot_build bu
JOIN runbot_branch br on (br.id = bu.branch_id)
JOIN runbot_repo r on (r.id = br.repo_id)
LEFT JOIN runbot_build du on (du.id = bu.duplicate_id and bu.state='duplicate')
WHERE br.sticky
AND br.repo_id in %s
AND (bu.hidden = 'f' OR bu.hidden IS NULL)
AND (
bu.state in ('running', 'done') or
(bu.state='duplicate' and du.state in ('running', 'done'))
bu.global_state in ('running', 'done')
)
AND coalesce(du.result, bu.result) not in ('skipped', 'manually_killed')
AND bu.global_result not in ('skipped', 'manually_killed')
GROUP BY 1,2,r.sequence,br.id
ORDER BY r.sequence, (br.branch_name='master'), br.id
"""
@ -335,14 +311,15 @@ class Runbot(Controller):
@route(['/runbot/branch/<int:branch_id>', '/runbot/branch/<int:branch_id>/page/<int:page>'], website=True, auth='public', type='http')
def branch_builds(self, branch_id=None, search='', page=1, limit=50, refresh='', **kwargs):
""" list builds of a runbot branch """
builds_count = request.env['runbot.build'].search_count([('branch_id','=',branch_id)])
domain =[('branch_id','=',branch_id), ('hidden', '=', False)]
builds_count = request.env['runbot.build'].search_count(domain)
pager = request.website.pager(
url='/runbot/branch/%s' % branch_id,
total=builds_count,
page=page,
step=50
)
builds = request.env['runbot.build'].search([('branch_id','=',branch_id)], limit=limit, offset=pager.get('offset',0))
builds = request.env['runbot.build'].search(domain, limit=limit, offset=pager.get('offset',0))
context = {'pager': pager, 'builds': builds}
return request.render("runbot.branch", context)

9
runbot/croninterval.py
View File

@ -1,9 +0,0 @@
# -*- coding: utf-8 -*-
import odoo
from dateutil.relativedelta import relativedelta
# increase cron frequency from 0.016 Hz to 0.1 Hz to reduce starvation and improve throughput with many workers
# TODO: find a nicer way than monkey patch to accomplish this
odoo.service.server.SLEEP_INTERVAL = 10
odoo.addons.base.ir.ir_cron._intervalTypes['minutes'] = lambda interval: relativedelta(seconds=interval * 10)

119
runbot/data/runbot_build_config_data.xml Normal file
View File

@ -0,0 +1,119 @@
<?xml version="1.0" encoding="utf-8"?>
<odoo>
<record id="runbot_build_config_step_test_base" model="runbot.build.config.step">
<field name="name">base</field>
<field name="install_modules">base</field>
<field name="cpu_limit">600</field>
<field name="test_enable" eval="False"/>
<field name="protected" eval="True"/>
<field name="default_sequence">10</field>
</record>
<record id="runbot_build_config_step_test_all" model="runbot.build.config.step">
<field name="name">all</field>
<field name="install_modules">*</field>
<field name="test_enable" eval="True"/>
<field name="protected" eval="True"/>
<field name="default_sequence">20</field>
</record>
<record id="runbot_build_config_step_run" model="runbot.build.config.step">
<field name="name">run</field>
<field name="job_type">run_odoo</field>
<field name="protected" eval="True"/>
<field name="default_sequence">1000</field>
</record>
<record id="runbot_build_config_default" model="runbot.build.config">
<field name="name">Default</field>
<field name="update_github_state" eval="True"/>
<field name="step_order_ids" eval="[(5,0,0),
(0, 0, {'step_id': ref('runbot_build_config_step_test_base')}),
(0, 0, {'step_id': ref('runbot_build_config_step_test_all')}),
(0, 0, {'step_id': ref('runbot_build_config_step_run')})]"/>
<field name="protected" eval="True"/>
</record>
<record id="runbot_build_config_default_no_run" model="runbot.build.config">
<field name="name">Default no run</field>
<field name="update_github_state" eval="True"/>
<field name="step_order_ids" eval="[(5,0,0),
(0, 0, {'step_id': ref('runbot_build_config_step_test_base')}),
(0, 0, {'step_id': ref('runbot_build_config_step_test_all')})]"/>
<field name="protected" eval="True"/>
</record>
<record id="runbot_build_config_light_test" model="runbot.build.config">
<field name="name">All only</field>
<field name="description">Test all only, usefull for multibuild</field>
<field name="step_order_ids" eval="[(5,0,0), (0, 0, {'step_id': ref('runbot_build_config_step_test_all')})]"/>
<field name="protected" eval="True"/>
</record>
<!-- Coverage-->
<record id="runbot_build_config_step_test_coverage" model="runbot.build.config.step">
<field name="name">coverage</field>
<field name="install_modules">*</field>
<field name="cpu_limit">5400</field>
<field name="test_enable" eval="True"/>
<field name="coverage" eval="True"/>
<field name="protected" eval="True"/>
<field name="default_sequence">30</field>
</record>
<record id="runbot_build_config_test_coverage" model="runbot.build.config">
<field name="name">Coverage</field>
<field name="step_order_ids" eval="[(5,0,0), (0, 0, {'step_id': ref('runbot_build_config_step_test_coverage')})]"/>
<field name="protected" eval="True"/>
</record>
<!-- Multi build-->
<record id="runbot_build_config_step_create_light_multi" model="runbot.build.config.step">
<field name="name">create_light_multi</field>
<field name="job_type">create_build</field>
<field name="create_config_ids" eval="[(4, ref('runbot_build_config_light_test'))]"/>
<field name="number_builds">10</field>
<field name="protected" eval="True"/>
</record>
<record id="runbot_build_config_multibuild" model="runbot.build.config">
<field name="name">Multi build</field>
<field name="description">Run 10 children build with the same hash and dependencies. Use to detect undeterministic issues</field>
<field name="step_order_ids" eval="[(5,0,0), (0, 0, {'step_id': ref('runbot_build_config_step_create_light_multi')})]"/>
<field name="protected" eval="True"/>
</record>
<!-- l10n todo check-->
<record id="runbot_build_config_step_test_l10n" model="runbot.build.config.step">
<field name="name">l10n</field>
<field name="install_modules">*</field>
<field name="test_enable" eval="True"/>
<field name="protected" eval="True"/>
<field name="default_sequence">30</field>
<field name="test_tags">l10nall</field>
<field name="protected" eval="True"/>
</record>
<record id="runbot_build_config_l10n" model="runbot.build.config">
<field name="name">L10n</field>
<field name="description">A simple test_all with a l10n test_tags</field>
<field name="step_order_ids" eval="[(5,0,0), (0, 0, {'step_id': ref('runbot_build_config_step_test_l10n')})]"/>
<field name="protected" eval="True"/>
</record>
<!-- Click all-->
<record id="runbot_build_config_step_test_click_all" model="runbot.build.config.step">
<field name="name">clickall</field>
<field name="install_modules">*</field>
<field name="cpu_limit">5400</field>
<field name="test_enable" eval="True"/>
<field name="protected" eval="True"/>
<field name="default_sequence">40</field>
<field name="test_tags">click_all</field>
<field name="protected" eval="True"/>
</record>
<record id="runbot_build_config_click_all" model="runbot.build.config">
<field name="name">Click All</field>
<field name="description">Used for nightly click all, test all filters and menus.</field>
<field name="step_order_ids" eval="[(5,0,0), (0, 0, {'step_id': ref('runbot_build_config_step_test_click_all')})]"/>
<field name="protected" eval="True"/>
</record>
</odoo>

2
runbot/models/__init__.py
View File

@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
from . import repo, branch, build, event, build_dependency
from . import repo, branch, build, event, build_dependency, build_config, ir_cron
from . import res_config_settings

45
runbot/models/branch.py
View File

@ -5,9 +5,9 @@ from subprocess import CalledProcessError
from odoo import models, fields, api
_logger = logging.getLogger(__name__)
_re_coverage = re.compile(r'\bcoverage\b')
_re_patch = re.compile(r'.*patch-\d+$')
class runbot_branch(models.Model):
_name = "runbot.branch"
@ -21,18 +21,26 @@ class runbot_branch(models.Model):
pull_head_name = fields.Char(compute='_get_branch_infos', string='PR HEAD name', readonly=1, store=True)
target_branch_name = fields.Char(compute='_get_branch_infos', string='PR target branch', readonly=1, store=True)
sticky = fields.Boolean('Sticky')
coverage = fields.Boolean('Coverage')
coverage_result = fields.Float(compute='_get_last_coverage', type='Float', string='Last coverage', store=False)
coverage_result = fields.Float(compute='_compute_coverage_result', type='Float', string='Last coverage', store=False) # non optimal search in loop, could we store this result ? or optimise
state = fields.Char('Status')
modules = fields.Char("Modules to Install", help="Comma-separated list of modules to install and test.")
job_timeout = fields.Integer('Job Timeout (minutes)', help='For default timeout: Mark it zero')
priority = fields.Boolean('Build priority', default=False)
job_type = fields.Selection([
('testing', 'Testing jobs only'),
('running', 'Running job only'),
('all', 'All jobs'),
('none', 'Do not execute jobs')
], required=True, default='all')
no_build = fields.Boolean("Forbid creation of build on this branch", default=False)
no_auto_build = fields.Boolean("Don't automatically build commit on this branch", default=False)
branch_config_id = fields.Many2one('runbot.build.config', 'Run Config')
config_id = fields.Many2one('runbot.build.config', 'Run Config', compute='_compute_config_id', inverse='_inverse_config_id')
def _compute_config_id(self):
for branch in self:
if branch.branch_config_id:
branch.config_id = branch.branch_config_id
else:
branch.config_id = branch.repo_id.config_id
def _inverse_config_id(self):
for branch in self:
branch.branch_config_id = branch.config_id
@api.depends('name')
def _get_branch_infos(self):
@ -76,8 +84,13 @@ class runbot_branch(models.Model):
return False
return True
@api.model
def create(self, vals):
vals.setdefault('coverage', _re_coverage.search(vals.get('name') or '') is not None)
if not vals.get('config_id') and ('use-coverage' in (vals.get('name') or '')):
coverage_config = self.env.ref('runbot.runbot_build_config_test_coverage', raise_if_not_found=False)
if coverage_config:
vals['config_id'] = coverage_config
return super(runbot_branch, self).create(vals)
def _get_branch_quickconnect_url(self, fqdn, dest):
@ -90,12 +103,12 @@ class runbot_branch(models.Model):
""" Return the last build with a coverage value > 0"""
self.ensure_one()
return self.env['runbot.build'].search([
('branch_id.id', '=', self.id),
('state', 'in', ['done', 'running']),
('coverage_result', '>=', 0.0),
], order='sequence desc', limit=1)
('branch_id.id', '=', self.id),
('local_state', 'in', ['done', 'running']),
('coverage_result', '>=', 0.0),
], order='sequence desc', limit=1)
def _get_last_coverage(self):
def _compute_coverage_result(self):
""" Compute the coverage result of the last build in branch """
for branch in self:
last_build = branch._get_last_coverage_build()

709
runbot/models/build.py
View File

@ -4,39 +4,31 @@ import logging
import os
import pwd
import re
import shlex
import shutil
import subprocess
import time
from ..common import dt2time, fqdn, now, grep, time2str, rfind, uniq_list, local_pgadmin_cursor, get_py_version
from ..container import docker_build, docker_run, docker_stop, docker_is_running, docker_get_gateway_ip, build_odoo_cmd
from ..common import dt2time, fqdn, now, grep, uniq_list, local_pgadmin_cursor, s2human
from ..container import docker_build, docker_stop, docker_is_running
from odoo import models, fields, api
from odoo.exceptions import UserError
from odoo.http import request
from odoo.tools import appdirs
_re_error = r'^(?:\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} \d+ (?:ERROR|CRITICAL) )|(?:Traceback \(most recent call last\):)$'
_re_warning = r'^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} \d+ WARNING '
re_job = re.compile('_job_\d')
from collections import defaultdict
from subprocess import CalledProcessError
_logger = logging.getLogger(__name__)
result_order = ['ok', 'warn', 'ko', 'skipped', 'killed', 'manually_killed']
state_order = ['pending', 'testing', 'waiting', 'running', 'deathrow', 'duplicate', 'done']
def runbot_job(*accepted_job_types):
""" Decorator for runbot_build _job_x methods to filter build jobs """
accepted_job_types += ('all', )
def job_decorator(func):
def wrapper(self, build, log_path):
if build.job_type == 'none' or build.job_type not in accepted_job_types:
build._log(func.__name__, 'Skipping job')
return -2
return func(self, build, log_path)
return wrapper
return job_decorator
def make_selection(array):
def format(string):
return (string, string.replace('_', ' ').capitalize())
return [format(elem) if isinstance(elem, str) else elem for elem in array]
class runbot_build(models.Model):
_name = "runbot.build"
_order = 'id desc'
@ -45,8 +37,8 @@ class runbot_build(models.Model):
name = fields.Char('Revno', required=True)
host = fields.Char('Host')
port = fields.Integer('Port')
dest = fields.Char(compute='_get_dest', type='char', string='Dest', readonly=1, store=True)
domain = fields.Char(compute='_get_domain', type='char', string='URL')
dest = fields.Char(compute='_compute_dest', type='char', string='Dest', readonly=1, store=True)
domain = fields.Char(compute='_compute_domain', type='char', string='URL')
date = fields.Datetime('Commit date')
author = fields.Char('Author')
author_email = fields.Char('Author Email')
@ -55,16 +47,31 @@ class runbot_build(models.Model):
subject = fields.Text('Subject')
sequence = fields.Integer('Sequence')
modules = fields.Char("Modules to Install")
result = fields.Char('Result', default='') # ok, ko, warn, skipped, killed, manually_killed
guess_result = fields.Char(compute='_guess_result')
# state machine
global_state = fields.Selection(make_selection(state_order), string='Status', compute='_compute_global_state', store=True)
local_state = fields.Selection(make_selection(state_order), string='Build Status', default='pending', required=True, oldname='state')
global_result = fields.Selection(make_selection(result_order), string='Result', compute='_compute_global_result', store=True)
local_result = fields.Selection(make_selection(result_order), string='Build Result', oldname='result')
triggered_result = fields.Selection(make_selection(result_order), string='Triggered Result') # triggered by db only
nb_pending = fields.Integer("Number of pending in queue", compute='_compute_nb_children', store=True)
nb_testing = fields.Integer("Number of test slot use", compute='_compute_nb_children', store=True)
nb_running = fields.Integer("Number of test slot use", compute='_compute_nb_children', store=True)
# should we add a stored field for children results?
pid = fields.Integer('Pid')
state = fields.Char('Status', default='pending') # pending, testing, running, done, duplicate, deathrow
job = fields.Char('Job') # job_*
active_step = fields.Many2one('runbot.build.config.step', 'Active step')
job = fields.Char('Active step display name', compute='_compute_job')
job_start = fields.Datetime('Job start')
job_end = fields.Datetime('Job end')
job_time = fields.Integer(compute='_get_time', string='Job time')
job_age = fields.Integer(compute='_get_age', string='Job age')
duplicate_id = fields.Many2one('runbot.build', 'Corresponding Build')
build_start = fields.Datetime('Build start')
build_end = fields.Datetime('Build end')
job_time = fields.Integer(compute='_compute_job_time', string='Job time')
build_time = fields.Integer(compute='_compute_build_time', string='Job time')
build_age = fields.Integer(compute='_compute_build_age', string='Build age')
duplicate_id = fields.Many2one('runbot.build', 'Corresponding Build', index=True)
server_match = fields.Selection([('builtin', 'This branch includes Odoo server'),
('match', 'This branch includes Odoo server'),
('default', 'No match found - defaults to master')],
@ -73,7 +80,7 @@ class runbot_build(models.Model):
column1='rev_dep_id', column2='dependent_id',
string='Builds that depends on this build')
extra_params = fields.Char('Extra cmd args')
coverage = fields.Boolean('Enable code coverage')
coverage = fields.Boolean('Code coverage was computed for this build')
coverage_result = fields.Float('Coverage result', digits=(5, 2))
build_type = fields.Selection([('scheduled', 'This build was automatically scheduled'),
('rebuild', 'This build is a rebuild'),
@ -82,22 +89,102 @@ class runbot_build(models.Model):
],
default='normal',
string='Build type')
job_type = fields.Selection([
('testing', 'Testing jobs only'),
('running', 'Running job only'),
('all', 'All jobs'),
('none', 'Do not execute jobs'),
])
parent_id = fields.Many2one('runbot.build', 'Parent Build', index=True)
# should we add a has children stored boolean?
hidden = fields.Boolean("Don't show build on main page", default=False) # index?
children_ids = fields.One2many('runbot.build', 'parent_id')
dependency_ids = fields.One2many('runbot.build.dependency', 'build_id')
config_id = fields.Many2one('runbot.build.config', 'Run Config', required=True, default=lambda self: self.env.ref('runbot.runbot_build_config_default', raise_if_not_found=False))
real_build = fields.Many2one('runbot.build', 'Real Build', help="duplicate_id or self", compute='_compute_real_build')
log_list = fields.Char('Comma separted list of step_ids names with logs', compute="_compute_log_list", store=True)
@api.depends('config_id')
def _compute_log_list(self): # storing this field beacause it will be access trhoug repo viewn and keep track of the list at create
for build in self:
build.log_list = ','.join({step.name for step in build.config_id.step_ids() if step._has_log()})
@api.depends('children_ids.global_state', 'local_state', 'duplicate_id.global_state')
def _compute_global_state(self):
for record in self:
if record.duplicate_id:
record.global_state = record.duplicate_id.global_state # not sure this is correct, redundant with real_build.global_state,
else:
waiting_score = record._get_state_score('waiting')
if record._get_state_score(record.local_state) > waiting_score and record.children_ids: # if finish, check children
children_state = record._get_youngest_state([child.global_state for child in record.children_ids])
# if all children are in running/done state (children could't be a duplicate I guess?)
if record._get_state_score(children_state) > waiting_score:
record.global_state = record.local_state
else:
record.global_state = 'waiting'
else:
record.global_state = record.local_state
def _get_youngest_state(self, states, max=False):
index = min([self._get_state_score(state) for state in states])
return state_order[index]
def _get_state_score(self, result):
return state_order.index(result)
# random note: need to count hidden in pending and testing build displayed in frontend
@api.depends('children_ids.global_result', 'local_result', 'duplicate_id.global_result')
def _compute_global_result(self):
for record in self: # looks like it's computed twice for children
if record.duplicate_id: # would like to avoid to add state as a depends only for this.
record.global_result = record.duplicate_id.global_result
elif not record.local_result:
record.global_result = record.local_result
elif record._get_result_score(record.local_result) >= record._get_result_score('ko'):
record.global_result = record.local_result
elif record.children_ids:
children_result = record._get_worst_result([child.global_result for child in record.children_ids], max_res='ko')
record.global_result = record._get_worst_result([record.local_result, children_result])
else:
record.global_result = record.local_result
def _get_worst_result(self, results, max_res=False):
results = [result for result in results if result] # filter Falsy values
index = max([self._get_result_score(result) for result in results]) if results else 0
if max_res:
return result_order[min([index, self._get_result_score(max_res)])]
return result_order[index]
def _get_result_score(self, result):
return result_order.index(result)
@api.depends('local_state', 'children_ids.nb_pending', 'children_ids.nb_testing', 'children_ids.nb_running')
def _compute_nb_children(self):
for record in self:
# note, we don't need to take care of duplicates, count will be 0 wich is right
record.nb_pending = int(record.local_state == 'pending')
record.nb_testing = int(record.local_state == 'testing')
record.nb_running = int(record.local_state == 'running')
for child in record.children_ids:
record.nb_pending += child.nb_pending
record.nb_testing += child.nb_testing
record.nb_running += child.nb_running
@api.depends('real_build.active_step')
def _compute_job(self):
for build in self:
build.job = build.real_build.active_step.name
@api.depends('duplicate_id')
def _compute_real_build(self):
for build in self:
build.real_build = build.duplicate_id or build
def copy(self, values=None):
raise UserError("Cannot duplicate build!")
def create(self, vals):
branch = self.env['runbot.branch'].search([('id', '=', vals.get('branch_id', False))])
if branch.job_type == 'none' or vals.get('job_type', '') == 'none':
branch = self.env['runbot.branch'].search([('id', '=', vals.get('branch_id', False))]) # branche 10174?
if branch.no_build:
return self.env['runbot.build']
vals['job_type'] = vals['job_type'] if 'job_type' in vals else branch.job_type
vals['config_id'] = vals['config_id'] if 'config_id' in vals else branch.config_id.id
build_id = super(runbot_build, self).create(vals)
extra_info = {'sequence': build_id.id if not build_id.sequence else build_id.sequence}
context = self.env.context
@ -106,18 +193,36 @@ class runbot_build(models.Model):
repo = build_id.repo_id
dep_create_vals = []
nb_deps = len(repo.dependency_ids)
for extra_repo in repo.dependency_ids:
(build_closets_branch, match_type) = build_id.branch_id._get_closest_branch(extra_repo.id)
closest_name = build_closets_branch.name
closest_branch_repo = build_closets_branch.repo_id
last_commit = closest_branch_repo._git_rev_parse(closest_name)
dep_create_vals.append({
'build_id': build_id.id,
'dependecy_repo_id': extra_repo.id,
'closest_branch_id': build_closets_branch.id,
'dependency_hash': last_commit,
'match_type': match_type,
})
params = build_id._get_params()
if not vals.get('dependency_ids'):
for extra_repo in repo.dependency_ids:
repo_name = extra_repo.short_name
last_commit = params['dep'][repo_name] # not name
if last_commit:
match_type = 'params'
build_closets_branch = False
message = 'Dependency for repo %s defined in commit message' % (repo_name)
else:
(build_closets_branch, match_type) = build_id.branch_id._get_closest_branch(extra_repo.id)
closest_name = build_closets_branch.name
closest_branch_repo = build_closets_branch.repo_id
last_commit = closest_branch_repo._git_rev_parse(closest_name)
message = 'Dependency for repo %s defined from closest branch %s' % (repo_name, closest_name)
try:
commit_oneline = extra_repo._git(['show', '--pretty="%H -- %s"', '-s', last_commit]).strip()
except CalledProcessError:
commit_oneline = 'Commit %s not found on build creation' % last_commit
# possible that oneline fail if given from commit message. Do it on build? or keep this information
build_id._log('create', '%s: %s' % (message, commit_oneline))
dep_create_vals.append({
'build_id': build_id.id,
'dependecy_repo_id': extra_repo.id,
'closest_branch_id': build_closets_branch.id,
'dependency_hash': last_commit,
'match_type': match_type,
})
for dep_vals in dep_create_vals:
self.env['runbot.build.dependency'].sudo().create(dep_vals)
@ -126,13 +231,13 @@ class runbot_build(models.Model):
# detect duplicate
duplicate_id = None
domain = [
('repo_id', 'in', (build_id.repo_id.duplicate_id.id, build_id.repo_id.id)), # before, was only looking in repo.duplicate_id looks a little better to search in both
('repo_id', 'in', (build_id.repo_id.duplicate_id.id, build_id.repo_id.id)), # before, was only looking in repo.duplicate_id looks a little better to search in both
('id', '!=', build_id.id),
('name', '=', build_id.name),
('duplicate_id', '=', False),
# ('build_type', '!=', 'indirect'), # in case of performance issue, this little fix may improve performance a little but less duplicate will be detected when pushing an empty branch on repo with duplicates
('result', '!=', 'skipped'),
('job_type', '=', build_id.job_type),
'|', ('local_result', '=', False), ('local_result', '!=', 'skipped'), # had to reintroduce False posibility for selections
('config_id', '=', build_id.config_id.id),
]
candidates = self.search(domain)
if candidates and nb_deps:
@ -164,79 +269,108 @@ class runbot_build(models.Model):
duplicate_id = candidates[0].id if candidates else False
if duplicate_id:
extra_info.update({'state': 'duplicate', 'duplicate_id': duplicate_id})
extra_info.update({'local_state': 'duplicate', 'duplicate_id': duplicate_id})
# maybe update duplicate priority if needed
build_id.write(extra_info)
if build_id.state == 'duplicate' and build_id.duplicate_id.state in ('running', 'done'):
if build_id.local_state == 'duplicate' and build_id.duplicate_id.global_state in ('running', 'done'): # and not build_id.parent_id:
build_id._github_status()
return build_id
def _reset(self):
self.write({'state': 'pending'})
def write(self, values):
# some validation to ensure db consistency
assert 'state' not in values
local_result = values.get('local_result')
assert not local_result or local_result == self._get_worst_result([self.local_result, local_result]) # dont write ok on a warn/error build
res = super(runbot_build, self).write(values)
assert bool(not self.duplicate_id) ^ (self.local_state == 'duplicate') # don't change duplicate state without removing duplicate id.
return res
def _end_test(self):
for build in self:
if build.parent_id:
global_result = build.global_result
loglevel = 'OK' if global_result == 'ok' else 'WARNING' if global_result == 'warn' else 'ERROR'
build.parent_id._log('children_build', 'returned a "%s" result ' % (global_result), level=loglevel, log_type='subbuild', path=self.id)
@api.depends('name', 'branch_id.name')
def _get_dest(self):
def _compute_dest(self):
for build in self:
nickname = build.branch_id.name.split('/')[2]
nickname = re.sub(r'"|\'|~|\:', '', nickname)
nickname = re.sub(r'_|/|\.', '-', nickname)
build.dest = ("%05d-%s-%s" % (build.id, nickname[:32], build.name[:6])).lower()
if build.name:
nickname = build.branch_id.name.split('/')[2]
nickname = re.sub(r'"|\'|~|\:', '', nickname)
nickname = re.sub(r'_|/|\.', '-', nickname)
build.dest = ("%05d-%s-%s" % (build.id, nickname[:32], build.name[:6])).lower()
def _get_domain(self):
@api.depends('repo_id', 'port', 'dest', 'host', 'duplicate_id.domain')
def _compute_domain(self):
domain = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_domain', fqdn())
for build in self:
if build.repo_id.nginx:
if build.duplicate_id:
build.domain = build.duplicate_id.domain
elif build.repo_id.nginx:
build.domain = "%s.%s" % (build.dest, build.host)
else:
build.domain = "%s:%s" % (domain, build.port)
def _guess_result(self):
cr = self.env.cr
cr.execute("""
SELECT b.id,
CASE WHEN array_agg(l.level)::text[] && ARRAY['ERROR', 'CRITICAL'] THEN 'ko'
WHEN array_agg(l.level)::text[] && ARRAY['WARNING'] THEN 'warn'
ELSE 'ok'
END
FROM runbot_build b
LEFT JOIN ir_logging l ON (l.build_id = b.id AND l.level != 'INFO')
WHERE b.id = ANY(%s)
GROUP BY b.id
""", [list(self.filtered(lambda b: b.state == 'testing').ids)])
result = {row[0]: row[1] for row in cr.fetchall()}
for build in self:
build.guess_result = result.get(build.id, build.result)
def _get_time(self):
@api.depends('job_start', 'job_end', 'duplicate_id.job_time')
def _compute_job_time(self):
"""Return the time taken by the tests"""
for build in self:
if build.job_end:
if build.duplicate_id:
build.job_time = build.duplicate_id.job_time
elif build.job_end:
build.job_time = int(dt2time(build.job_end) - dt2time(build.job_start))
elif build.job_start:
build.job_time = int(time.time() - dt2time(build.job_start))
def _get_age(self):
@api.depends('build_start', 'build_end', 'duplicate_id.build_time')
def _compute_build_time(self):
for build in self:
if build.duplicate_id:
build.build_time = build.duplicate_id.build_time
elif build.build_end:
build.build_time = int(dt2time(build.build_end) - dt2time(build.build_start))
elif build.build_start:
build.build_time = int(time.time() - dt2time(build.build_start))
@api.depends('job_start', 'duplicate_id.build_age')
def _compute_build_age(self):
"""Return the time between job start and now"""
for build in self:
if build.job_start:
build.job_age = int(time.time() - dt2time(build.job_start))
if build.duplicate_id:
build.build_age = build.duplicate_id.build_age
elif build.job_start:
build.build_age = int(time.time() - dt2time(build.build_start))
def _get_params(self):
message = False
try:
message = self.repo_id._git(['show', '-s', self.name])
except CalledProcessError:
pass # todo remove this try catch and make correct patch for _git
params = defaultdict(lambda: defaultdict(str))
if message:
regex = re.compile(r'^[\t ]*dep=([A-Za-z0-9\-_]+/[A-Za-z0-9\-_]+):([0-9A-Fa-f\-]*) *(#.*)?$', re.M) # dep:repo:hash #comment
for result in re.findall(regex, message):
params['dep'][result[0]] = result[1]
return params
def _force(self, message=None):
"""Force a rebuild and return a recordset of forced builds"""
forced_builds = self.env['runbot.build']
for build in self:
pending_ids = self.search([('state', '=', 'pending')], order='id', limit=1)
pending_ids = self.search([('local_state', '=', 'pending')], order='id', limit=1)
if pending_ids:
sequence = pending_ids[0].id
else:
sequence = self.search([], order='id desc', limit=1)[0].id
# Force it now
rebuild = True
if build.state == 'done' and build.result == 'skipped':
build.write({'state': 'pending', 'sequence': sequence, 'result': ''})
if build.local_state == 'done' and build.local_result == 'skipped':
build.write({'local_state': 'pending', 'sequence': sequence, 'local_result': ''})
# or duplicate it
elif build.state in ['running', 'done', 'duplicate', 'deathrow']:
elif build.local_state in ['running', 'done', 'duplicate', 'deathrow']:
new_build = build.with_context(force_rebuild=True).create({
'sequence': sequence,
'branch_id': build.branch_id.id,
@ -247,7 +381,10 @@ class runbot_build(models.Model):
'committer_email': build.committer_email,
'subject': build.subject,
'modules': build.modules,
'build_type': 'rebuild'
'build_type': 'rebuild',
# 'config_id': build.config_id.id,
# we use the branch config for now since we are recomputing dependencies,
# we may introduce an 'exact rebuild' later
})
build = new_build
else:
@ -264,9 +401,7 @@ class runbot_build(models.Model):
"""Mark builds ids as skipped"""
if reason:
self._logger('skip %s', reason)
self.write({'state': 'done', 'result': 'skipped'})
to_unduplicate = self.search([('id', 'in', self.ids), ('duplicate_id', '!=', False)])
to_unduplicate._force()
self.write({'local_state': 'done', 'local_result': 'skipped', 'duplicate_id': False})
def _local_cleanup(self):
for build in self:
@ -291,8 +426,8 @@ class runbot_build(models.Model):
SELECT dest
FROM runbot_build
WHERE dest IN %s
AND (state != 'done' OR job_end > (now() - interval '7 days'))
""", [tuple(builds)])
AND (local_state != 'done' OR job_end > (now() - interval '7 days'))
""", [tuple(builds)]) # todo xdo not covered by tests
actives = set(b[0] for b in self.env.cr.fetchall())
for b in builds:
@ -301,7 +436,7 @@ class runbot_build(models.Model):
shutil.rmtree(path)
# cleanup old unused databases
self.env.cr.execute("select id from runbot_build where state in ('testing', 'running')")
self.env.cr.execute("select id from runbot_build where local_state in ('testing', 'running')") # todo xdo not coversed by tests
db_ids = [id[0] for id in self.env.cr.fetchall()]
if db_ids:
with local_pgadmin_cursor() as local_cr:
@ -317,13 +452,9 @@ class runbot_build(models.Model):
for db, in to_delete:
self._local_pg_dropdb(db)
def _list_jobs(self):
"""List methods that starts with _job_[[:digit:]]"""
return sorted(job[1:] for job in dir(self) if re_job.match(job))
def _find_port(self):
# currently used port
ids = self.search([('state', 'not in', ['pending', 'done'])])
ids = self.search([('local_state', 'not in', ['pending', 'done'])])
ports = set(i['port'] for i in ids.read(['port']))
# starting port
@ -343,88 +474,114 @@ class runbot_build(models.Model):
def _get_docker_name(self):
self.ensure_one()
return '%s_%s' % (self.dest, self.job)
return '%s_%s' % (self.dest, self.active_step.name)
def _schedule(self):
"""schedule the build"""
jobs = self._list_jobs()
icp = self.env['ir.config_parameter']
# For retro-compatibility, keep this parameter in seconds
default_timeout = int(icp.get_param('runbot.runbot_timeout', default=3600)) / 60
for build in self:
if build.state == 'deathrow':
if build.local_state == 'deathrow':
build._kill(result='manually_killed')
continue
elif build.state == 'pending':
if build.local_state == 'pending':
# allocate port and schedule first job
port = self._find_port()
values = {
'host': fqdn(),
'host': fqdn(), # or ip? of false?
'port': port,
'state': 'testing',
'job': jobs[0],
'job_start': now(),
'build_start': now(),
'job_end': False,
}
values.update(build._next_job_values())
build.write(values)
else:
if not build.active_step:
build._log('schedule', 'No job in config, doing nothing')
build._end_test()
continue
try:
build._log('init', 'Init build environment with config %s ' % build.config_id.name)
# notify pending build - avoid confusing users by saying nothing
build._github_status()
build._checkout()
build._log('docker_build', 'Building docker image')
docker_build(build._path('logs', 'docker_build.txt'), build._path())
except Exception:
_logger.exception('Failed initiating build %s', build.dest)
build._log('Failed initiating build')
build._kill(result='ko')
continue
else: # testing/running build
if build.local_state == 'testing':
# failfast in case of docker error (triggered in database)
if (not build.local_result or build.local_result == 'ok') and build.triggered_result:
build.local_result = build.triggered_result
build._github_status() # failfast
# check if current job is finished
if docker_is_running(build._get_docker_name()):
# kill if overpassed
timeout = (build.branch_id.job_timeout or default_timeout) * 60 * ( build.coverage and 1.5 or 1)
if build.job != jobs[-1] and build.job_time > timeout:
build._log('schedule', '%s time exceeded (%ss)' % (build.job, build.job_time))
timeout = min(build.active_step.cpu_limit, int(icp.get_param('runbot.runbot_timeout', default=10000)))
if build.local_state != 'running' and build.job_time > timeout:
build._logger('%s time exceded (%ss)', build.active_step.name if build.active_step else "?", build.job_time)
build.write({'job_end': now()})
build._kill(result='killed')
else:
# failfast
if not build.result and build.guess_result in ('ko', 'warn'):
build.result = build.guess_result
build._github_status()
continue
build._logger('%s finished', build.job)
# schedule
v = {}
# testing -> running
if build.job == jobs[-2]:
v['state'] = 'running'
v['job'] = jobs[-1]
v['job_end'] = now(),
# running -> done
elif build.job == jobs[-1]:
v['state'] = 'done'
v['job'] = ''
# testing
# No job running, make result and select nex job
build_values = {
'job_end': now(),
}
# make result of previous job
try:
results = build.active_step._make_results(build)
except:
_logger.exception('An error occured while computing results')
build._log('_make_results', 'An error occured while computing results', level='ERROR')
results = {'local_state': 'ko'}
build_values.update(results)
# Non running build in
notify_end_job = build.active_step.job_type != 'create_build'
build_values.update(build._next_job_values()) # find next active_step or set to done
ending_build = build.local_state not in ('done', 'running') and build_values.get('local_state') in ('done', 'running')
if ending_build:
build_values['build_end'] = now()
step_end_message = 'Step %s finished in %s' % (build.job, s2human(build.job_time))
build.write(build_values)
if ending_build:
build._github_status()
build._end_test() # notify parent of build end
if not build.local_result: # Set 'ok' result if no result set (no tests job on build)
build.local_result = 'ok'
build._logger("No result set, setting ok by default")
if notify_end_job:
build._log('end_job', step_end_message)
else:
v['job'] = jobs[jobs.index(build.job) + 1]
build.write(v)
build._logger(step_end_message)
# run job
pid = None
if build.state != 'done':
build._logger('running %s', build.job)
job_method = getattr(self, '_' + build.job) # compute the job method to run
if build.local_state != 'done':
build._logger('running %s', build.active_step.name)
os.makedirs(build._path('logs'), exist_ok=True)
os.makedirs(build._path('datadir'), exist_ok=True)
log_path = build._path('logs', '%s.txt' % build.job)
try:
pid = job_method(build, log_path)
build.write({'pid': pid})
except Exception:
_logger.exception('%s failed running method %s', build.dest, build.job)
build._log(build.job, "failed running job method, see runbot log")
pid = build.active_step._run(build) # run should be on build?
build.write({'pid': pid}) # no really usefull anymore with dockers
except Exception as e:
message = '%s failed running step %s' % (build.dest, build.job)
_logger.exception(message)
build._log("run", message)
build._kill(result='ko')
continue
if pid == -2:
# no process to wait, directly call next job
# FIXME find a better way that this recursive call
build._schedule()
# cleanup only needed if it was not killed
if build.state == 'done':
if build.local_state == 'done':
build._local_cleanup()
def _path(self, *l, **kw):
@ -434,7 +591,7 @@ class runbot_build(models.Model):
root = self.env['runbot.repo']._root()
return os.path.join(root, 'build', build.dest, *l)
def _server(self, *l, **kw):
def _server(self, *l, **kw): # not really build related, specific to odoo version, could be a data
"""Return the build server path"""
self.ensure_one()
build = self
@ -456,7 +613,7 @@ class runbot_build(models.Model):
return uniq_list(filter(mod_filter, modules))
def _checkout(self):
self.ensure_one() # will raise exception if hash not found, we don't want to fail for all build.
self.ensure_one() # will raise exception if hash not found, we don't want to fail for all build.
# starts from scratch
build = self
if os.path.isdir(build._path()):
@ -497,30 +654,25 @@ class runbot_build(models.Model):
for build_dependency in build.dependency_ids:
closest_branch = build_dependency.closest_branch_id
latest_commit = build_dependency.dependency_hash
repo = closest_branch.repo_id
closest_name = closest_branch.name
repo = closest_branch.repo_id or build_dependency.repo_id
closest_name = closest_branch.name or 'no_branch'
if build_dependency.match_type == 'default':
server_match = 'default'
elif server_match != 'default':
server_match = 'match'
build._log(
'Building environment',
'%s match branch %s of %s' % (build_dependency.match_type, closest_name, repo.name)
'_checkout', 'Checkouting %s from %s' % (closest_name, repo.name)
)
if not repo._hash_exists(latest_commit):
repo._update(force=True)
repo._update(repo, force=True)
if not repo._hash_exists(latest_commit):
repo._git(['fetch', 'origin', latest_commit])
if not repo._hash_exists(latest_commit):
build._log('_checkout',"Dependency commit %s in repo %s is unreachable" % (latest_commit, repo.name))
build._log('_checkout', "Dependency commit %s in repo %s is unreachable" % (latest_commit, repo.name))
raise Exception
commit_oneline = repo._git(['show', '--pretty="%H -- %s"', '-s', latest_commit]).strip()
build._log(
'Building environment',
'Server built based on commit %s from %s' % (commit_oneline, closest_name)
)
repo._git_export(latest_commit, build._path())
# Finally mark all addons to move to openerp/addons
@ -578,23 +730,20 @@ class runbot_build(models.Model):
with local_pgadmin_cursor() as local_cr:
local_cr.execute("""CREATE DATABASE "%s" TEMPLATE template0 LC_COLLATE 'C' ENCODING 'unicode'""" % dbname)
def _log(self, func, message):
def _log(self, func, message, level='INFO', log_type='runbot', path='runbot'):
self.ensure_one()
_logger.debug("Build %s %s %s", self.id, func, message)
self.env['ir.logging'].create({
'build_id': self.id,
'level': 'INFO',
'type': 'runbot',
'level': level,
'type': log_type,
'name': 'odoo.runbot',
'message': message,
'path': 'runbot',
'path': path,
'func': func,
'line': '0',
})
def reset(self):
self.write({'state': 'pending'})
def _reap(self):
while True:
try:
@ -612,27 +761,34 @@ class runbot_build(models.Model):
continue
build._log('kill', 'Kill build %s' % build.dest)
docker_stop(build._get_docker_name())
v = {'state': 'done', 'job': False}
v = {'local_state': 'done', 'active_step': False, 'duplicate': False} # what if duplicate? state done?
if result:
v['result'] = result
v['local_result'] = result
build.write(v)
self.env.cr.commit()
build._github_status()
build._local_cleanup()
def _ask_kill(self):
# todo xdo, should we kill or skip children builds? it looks like yes, but we need to be carefull if subbuild can be duplicates
self.ensure_one()
user = request.env.user if request else self.env.user
uid = user.id
build = self.duplicate_id if self.state == 'duplicate' else self
if build.state == 'pending':
build = self
if build.duplicate_id:
if build.duplicate_id.branch_id.sticky:
build._skip()
build._log('_ask_kill', 'Skipping build %s, requested by %s (user #%s)(duplicate of sticky, kill duplicate)' % (build.dest, user.name, uid))
return
build = build.duplicate_id # if duplicate is not sticky, most likely a pr, kill other build
if build.local_state == 'pending':
build._skip()
build._log('_ask_kill', 'Skipping build %s, requested by %s (user #%s)' % (build.dest, user.name, uid))
elif build.state in ['testing', 'running']:
build.write({'state': 'deathrow'})
elif build.local_state in ['testing', 'running']:
build.write({'local_state': 'deathrow'})
build._log('_ask_kill', 'Killing build %s, requested by %s (user #%s)' % (build.dest, user.name, uid))
def _cmd(self):
def _cmd(self): # why not remove build.modules output ?
"""Return a tuple describing the command to start the build
First part is list with the command and parameters
Second part is a list of Odoo modules
@ -652,7 +808,7 @@ class runbot_build(models.Model):
# commandline
cmd = [ os.path.join('/data/build', odoo_bin), ]
# options
if grep(build._server("tools/config.py"), "no-xmlrpcs"):
if grep(build._server("tools/config.py"), "no-xmlrpcs"): # move that to configs ?
cmd.append("--no-xmlrpcs")
if grep(build._server("tools/config.py"), "no-netrpc"):
cmd.append("--no-netrpc")
@ -676,166 +832,65 @@ class runbot_build(models.Model):
return cmd, build.modules
def _github_status_notify_all(self, status):
"""Notify each repo with a status"""
self.ensure_one()
commits = {(b.repo_id, b.name) for b in self.search([('name', '=', self.name)])}
for repo, commit_hash in commits:
_logger.debug("github updating %s status %s to %s in repo %s", status['context'], commit_hash, status['state'], repo.name)
repo._github('/repos/:owner/:repo/statuses/%s' % commit_hash, status, ignore_errors=True)
if self.config_id.update_github_state:
commits = {(b.repo_id, b.name) for b in self.search([('name', '=', self.name)])}
for repo, commit_hash in commits:
_logger.debug("github updating %s status %s to %s in repo %s", status['context'], commit_hash, status['state'], repo.name)
repo._github('/repos/:owner/:repo/statuses/%s' % commit_hash, status, ignore_errors=True)
def _github_status(self):
"""Notify github of failed/successful builds"""
runbot_domain = self.env['runbot.repo']._domain()
for build in self:
desc = "runbot build %s" % (build.dest,)
if build.state == 'testing':
state = 'pending'
elif build.state in ('running', 'done'):
state = 'error'
else:
continue
desc += " (runtime %ss)" % (build.job_time,)
if build.result == 'ok':
state = 'success'
if build.result in ('ko', 'warn'):
state = 'failure'
status = {
"state": state,
"target_url": "http://%s/runbot/build/%s" % (runbot_domain, build.id),
"description": desc,
"context": "ci/runbot"
}
build._github_status_notify_all(status)
if build.config_id.update_github_state:
runbot_domain = self.env['runbot.repo']._domain()
desc = "runbot build %s" % (build.dest,)
if build.local_state == 'testing':
state = 'pending'
elif build.local_state in ('running', 'done'):
state = 'error'
else:
continue
desc += " (runtime %ss)" % (build.job_time,)
if build.local_result == 'ok':
state = 'success'
if build.local_result in ('ko', 'warn'):
state = 'failure'
status = {
"state": state,
"target_url": "http://%s/runbot/build/%s" % (runbot_domain, build.id),
"description": desc,
"context": "ci/runbot"
}
build._github_status_notify_all(status)
# Jobs definitions
# They all need "build log_path" parameters
@runbot_job('testing', 'running')
def _job_00_init(self, build, log_path):
build._log('init', 'Init build environment')
# notify pending build - avoid confusing users by saying nothing
build._github_status()
build._checkout()
return -2
def _next_job_values(self):
self.ensure_one()
step_ids = self.config_id.step_ids()
if not step_ids: # no job to do, build is done
return {'active_step': False, 'local_state': 'done'}
@runbot_job('testing', 'running')
def _job_02_docker_build(self, build, log_path):
"""Build the docker image"""
build._log('docker_build', 'Building docker image')
docker_build(log_path, build._path())
return -2
next_index = step_ids.index(self.active_step) + 1 if self.active_step else 0
if next_index >= len(step_ids): # final job, build is done
return {'active_step': False, 'local_state': 'done'}
@runbot_job('testing')
def _job_10_test_base(self, build, log_path):
build._log('test_base', 'Start test base module')
self._local_pg_createdb("%s-base" % build.dest)
cmd, mods = build._cmd()
cmd += ['-d', '%s-base' % build.dest, '-i', 'base', '--stop-after-init', '--log-level=test', '--max-cron-threads=0']
if build.extra_params:
cmd.extend(shlex.split(build.extra_params))
return docker_run(build_odoo_cmd(cmd), log_path, build._path(), build._get_docker_name(), cpu_limit=600)
new_step = step_ids[next_index] # job to do, state is job_state (testing or running)
return {'active_step': new_step.id, 'local_state': new_step._step_state()}
@runbot_job('testing', 'running')
def _job_20_test_all(self, build, log_path):
def build_type_label(self):
self.ensure_one()
return dict(self.fields_get('build_type', 'selection')['build_type']['selection']).get(self.build_type, self.build_type)
cpu_limit = int(self.env['ir.config_parameter'].get_param('runbot.runbot_timeout', default=3600))
self._local_pg_createdb("%s-all" % build.dest)
cmd, mods = build._cmd()
build._log('test_all', 'Start test all modules')
if grep(build._server("tools/config.py"), "test-enable") and build.job_type in ('testing', 'all'):
cmd.extend(['--test-enable', '--log-level=test'])
else:
build._log('test_all', 'Installing modules without testing')
cmd += ['-d', '%s-all' % build.dest, '-i', mods, '--stop-after-init', '--max-cron-threads=0']
if build.extra_params:
cmd.extend(build.extra_params.split(' '))
if build.coverage:
cpu_limit *= 1.5
available_modules = [
os.path.basename(os.path.dirname(a))
for a in (glob.glob(build._server('addons/*/__openerp__.py')) +
glob.glob(build._server('addons/*/__manifest__.py')))
]
bad_modules = set(available_modules) - set((mods or '').split(','))
omit = ['--omit', ','.join('*addons/%s/*' %m for m in bad_modules) + '*__manifest__.py']
cmd = [ get_py_version(build), '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + omit + cmd
# reset job_start to an accurate job_20 job_time
build.write({'job_start': now()})
return docker_run(build_odoo_cmd(cmd), log_path, build._path(), build._get_docker_name(), cpu_limit=cpu_limit)
def get_formated_job_time(self):
return s2human(self.job_time)
@runbot_job('testing')
def _job_21_coverage_html(self, build, log_path):
if not build.coverage:
return -2
build._log('coverage_html', 'Start generating coverage html')
cov_path = build._path('coverage')
os.makedirs(cov_path, exist_ok=True)
cmd = [ get_py_version(build), "-m", "coverage", "html", "-d", "/data/build/coverage", "--ignore-errors"]
return docker_run(build_odoo_cmd(cmd), log_path, build._path(), build._get_docker_name())
def get_formated_build_time(self):
return s2human(self.build_time)
@runbot_job('testing')
def _job_22_coverage_result(self, build, log_path):
if not build.coverage:
return -2
build._log('coverage_result', 'Start getting coverage result')
cov_path = build._path('coverage/index.html')
if os.path.exists(cov_path):
with open(cov_path,'r') as f:
data = f.read()
covgrep = re.search(r'pc_cov.>(?P<coverage>\d+)%', data)
build.coverage_result = covgrep and covgrep.group('coverage') or False
else:
build._log('coverage_result', 'Coverage file not found')
return -2 # nothing to wait for
def get_formated_build_age(self):
return s2human(self.build_age)
@runbot_job('testing', 'running')
def _job_29_results(self, build, log_path):
build._log('run', 'Getting results for build %s' % build.dest)
log_all = build._path('logs', 'job_20_test_all.txt')
log_time = time.localtime(os.path.getmtime(log_all))
v = {
'job_end': time2str(log_time),
}
if grep(log_all, ".modules.loading: Modules loaded."):
if rfind(log_all, _re_error):
v['result'] = "ko"
elif rfind(log_all, _re_warning):
v['result'] = "warn"
elif not grep(log_all, "Initiating shutdown"):
v['result'] = "ko"
build._log('run', "Seems that the build was stopped too early. The cpu_limit could have been reached")
elif not build.result:
v['result'] = "ok"
else:
build._log('run', "Modules not loaded")
v['result'] = "ko"
build.write(v)
build._github_status()
return -2
@runbot_job('running')
def _job_30_run(self, build, log_path):
# adjust job_end to record an accurate job_20 job_time
build._log('run', 'Start running build %s' % build.dest)
# run server
cmd, mods = build._cmd()
if os.path.exists(build._server('addons/im_livechat')):
cmd += ["--workers", "2"]
cmd += ["--longpolling-port", "8070"]
cmd += ["--max-cron-threads", "1"]
else:
# not sure, to avoid old server to check other dbs
cmd += ["--max-cron-threads", "0"]
cmd += ['-d', '%s-all' % build.dest]
if grep(build._server("tools/config.py"), "db-filter"):
if build.repo_id.nginx:
cmd += ['--db-filter', '%d.*$']
else:
cmd += ['--db-filter', '%s.*$' % build.dest]
smtp_host = docker_get_gateway_ip()
if smtp_host:
cmd += ['--smtp', smtp_host]
return docker_run(build_odoo_cmd(cmd), log_path, build._path(), build._get_docker_name(), exposed_ports = [build.port, build.port + 1])
def sorted_revdep_build_ids(self):
return sorted(self.revdep_build_ids, key=lambda build: build.repo_id.name)

396
runbot/models/build_config.py Normal file
View File

@ -0,0 +1,396 @@
import glob
import logging
import os
import re
import shlex
import time
from ..common import now, grep, get_py_version, time2str, rfind
from ..container import docker_run, docker_get_gateway_ip, build_odoo_cmd
from odoo import models, fields, api
from odoo.exceptions import UserError, ValidationError
from odoo.tools.safe_eval import safe_eval, test_python_expr
_logger = logging.getLogger(__name__)
_re_error = r'^(?:\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} \d+ (?:ERROR|CRITICAL) )|(?:Traceback \(most recent call last\):)$'
_re_warning = r'^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} \d+ WARNING '
class Config(models.Model):
_name = "runbot.build.config"
_inherit = "mail.thread"
name = fields.Char('Config name', required=True, unique=True, tracking=True, help="Unique name for config please use trigram as postfix for custom configs")
description = fields.Char('Config description')
step_order_ids = fields.One2many('runbot.build.config.step.order', 'config_id')
update_github_state = fields.Boolean('Notify build state to github', default=False, tracking=True)
protected = fields.Boolean('Protected', default=False, tracking=True)
@api.model
def create(self, values):
res = super(Config, self).create(values)
res._check_step_ids_order()
return res
def write(self, values):
res = super(Config, self).write(values)
self._check_step_ids_order()
return res
def copy(self):
# remove protection on copy
copy = super(Config, self).copy()
copy.sudo().write({'protected': False})
return copy
def unlink(self):
super(Config, self).unlink()
def step_ids(self):
self.ensure_one()
return [ordered_step.step_id for ordered_step in self.step_order_ids]
def _check_step_ids_order(self):
install_job = False
step_ids = self.step_ids()
for step in step_ids:
if step.job_type == 'install_odoo':
install_job = True
if step.job_type == 'run_odoo':
if step != step_ids[-1]:
raise UserError('Jobs of type run_odoo should be the last one')
if not install_job:
raise UserError('Jobs of type run_odoo should be preceded by a job of type install_odoo')
self._check_recustion()
def _check_recustion(self, visited=None): # todo test
visited = visited or []
recursion = False
if self in visited:
recursion = True
visited.append(self)
if recursion:
raise UserError('Impossible to save config, recursion detected with path: %s' % ">".join([v.name for v in visited]))
for step in self.step_ids():
if step.job_type == 'create_build':
for create_config in step.create_config_ids:
create_config._check_recustion(visited[:])
class ConfigStep(models.Model):
_name = 'runbot.build.config.step'
_inherit = 'mail.thread'
# general info
name = fields.Char('Step name', required=True, unique=True, tracking=True, help="Unique name for step please use trigram as postfix for custom step_ids")
job_type = fields.Selection([
('install_odoo', 'Test odoo'),
('run_odoo', 'Run odoo'),
('python', 'Python code'),
('create_build', 'Create build'),
], default='install_odoo', required=True, tracking=True)
protected = fields.Boolean('Protected', default=False, tracking=True)
default_sequence = fields.Integer('Sequence', default=100, tracking=True) # or run after? # or in many2many rel?
# install_odoo
create_db = fields.Boolean('Create Db', default=True, tracking=True) # future
custom_db_name = fields.Char('Custom Db Name', tracking=True) # future
install_modules = fields.Char('Modules to install', help="List of module to install, use * for all modules")
db_name = fields.Char('Db Name', compute='_compute_db_name', inverse='_inverse_db_name', tracking=True)
cpu_limit = fields.Integer('Cpu limit', default=3600, tracking=True)
coverage = fields.Boolean('Coverage', dafault=False, tracking=True)
test_enable = fields.Boolean('Test enable', default=False, tracking=True)
test_tags = fields.Char('Test tags', help="comma separated list of test tags")
extra_params = fields.Char('Extra cmd args', tracking=True)
# python
python_code = fields.Text('Python code', tracking=True, default="# type python code here\n\n\n\n\n\n")
running_job = fields.Boolean('Job final state is running', default=False, help="Docker won't be killed if checked")
# create_build
create_config_ids = fields.Many2many('runbot.build.config', 'runbot_build_config_step_ids_create_config_ids_rel', string='New Build Configs', tracking=True, index=True)
number_builds = fields.Integer('Number of build to create', default=1, tracking=True)
hide_build = fields.Boolean('Hide created build in frontend', default=True, tracking=True)
force_build = fields.Boolean("As a forced rebuild, don't use duplicate detection", default=False, tracking=True)
force_host = fields.Boolean('Use same host as parent for children', default=False, tracking=True) # future
@api.constrains('python_code')
def _check_python_code(self):
for step in self.sudo().filtered('python_code'):
msg = test_python_expr(expr=step.python_code.strip(), mode="exec")
if msg:
raise ValidationError(msg)
@api.depends('name', 'custom_db_name')
def _compute_db_name(self):
for step in self:
step.db_name = step.custom_db_name or step.name
def _inverse_db_name(self):
for step in self:
step.custom_db_name = step.db_name
def copy(self):
# remove protection on copy
copy = super(ConfigStep, self).copy()
copy._write({'protected': False})
return copy
@api.model
def create(self, values):
self._check(values)
return super(ConfigStep, self).create(values)
def write(self, values):
self._check(values)
return super(ConfigStep, self).write(values)
def unlink(self):
if self.protected:
raise UserError('Protected step')
super(ConfigStep, self).unlink()
def _check(self, values):
if 'name' in values:
name_reg = r'^[a-zA-Z0-9\-_]*$'
if not re.match(name_reg, values.get('name')):
raise UserError('Name cannot contain special char or spaces exepts "_" and "-"')
if not self.env.user.has_group('runbot.group_build_config_administrator'):
if (values.get('job_type') == 'python' or ('python_code' in values and values['python_code'])):
raise UserError('cannot create or edit config step of type python code')
if (values.get('extra_params')):
reg = r'^[a-zA-Z0-9\-_ "]*$'
if not re.match(reg, values.get('extra_params')):
_logger.log('%s tried to create an non supported test_param %s' % (self.env.user.name, values.get('extra_params')))
raise UserError('Invalid extra_params on config step')
def _run(self, build):
log_path = build._path('logs', '%s.txt' % self.name)
build.write({'job_start': now(), 'job_end': False}) # state, ...
build._log('run', 'Starting step %s from config %s' % (self.name, build.config_id.name), level='SEPARATOR')
return self._run_step(log_path, build)
def _run_step(self, log_path, build):
if self.job_type == 'run_odoo':
return self._run_odoo_run(build, log_path)
if self.job_type == 'install_odoo':
return self._run_odoo_install(build, log_path)
elif self.job_type == 'python':
return self._run_python(build, log_path)
elif self.job_type == 'create_build':
return self._create_build(build, log_path)
def _create_build(self, build, log_path):
Build = self.env['runbot.build']
if self.force_build:
Build = Build.with_context(force_rebuild=True)
count = 0
for create_config in self.create_config_ids:
for _ in range(self.number_builds):
count += 1
if count > 200:
build._logger('Too much build created')
break
children = Build.create({
'dependency_ids': [(4, did.id) for did in build.dependency_ids],
'config_id': create_config.id,
'parent_id': build.id,
'branch_id': build.branch_id.id,
'name': build.name,
'build_type': build.build_type,
'date': build.date,
'author': build.author,
'author_email': build.author_email,
'committer': build.committer,
'committer_email': build.committer_email,
'subject': build.subject,
'modules': build.modules,
'hidden': self.hide_build,
})
build._log('create_build', 'created with config %s' % create_config.name, log_type='subbuild', path=str(children.id))
def _run_python(self, build, log_path):
eval_ctx = {'self': self, 'build': build, 'log_path': log_path}
return safe_eval(self.sudo().code.strip(), eval_ctx, mode="exec", nocopy=True)
def _run_odoo_run(self, build, log_path):
# adjust job_end to record an accurate job_20 job_time
build._log('run', 'Start running build %s' % build.dest)
# run server
cmd, _ = build._cmd()
if os.path.exists(build._server('addons/im_livechat')):
cmd += ["--workers", "2"]
cmd += ["--longpolling-port", "8070"]
cmd += ["--max-cron-threads", "1"]
else:
# not sure, to avoid old server to check other dbs
cmd += ["--max-cron-threads", "0"]
db_name = [step.db_name for step in build.config_id.step_ids() if step.job_type == 'install_odoo'][-1]
# we need to have at least one job of type install_odoo to run odoo, take the last one for db_name.
cmd += ['-d', '%s-%s' % (build.dest, db_name)]
if grep(build._server("tools/config.py"), "db-filter"):
if build.repo_id.nginx:
cmd += ['--db-filter', '%d.*$']
else:
cmd += ['--db-filter', '%s.*$' % build.dest]
smtp_host = docker_get_gateway_ip()
if smtp_host:
cmd += ['--smtp', smtp_host]
return docker_run(build_odoo_cmd(cmd), log_path, build._path(), build._get_docker_name(), exposed_ports=[build.port, build.port + 1])
def _run_odoo_install(self, build, log_path):
cmd, _ = build._cmd()
# create db if needed
db_name = "%s-%s" % (build.dest, self.db_name)
if self.create_db:
build._local_pg_createdb(db_name)
cmd += ['-d', db_name]
# list module to install
modules_to_install = self._modules_to_install(build)
mods = ",".join(modules_to_install)
if mods:
cmd += ['-i', mods]
if self.test_enable:
if grep(build._server("tools/config.py"), "test-enable"):
cmd.extend(['--test-enable'])
else:
build._log('test_all', 'Installing modules without testing', level='WARNING')
if self.test_tags:
test_tags = self.test_tags.replace(' ', '')
cmd.extend(['--test-tags', test_tags])
cmd += ['--stop-after-init'] # install job should always finish
cmd += ['--log-level=test', '--max-cron-threads=0']
if self.extra_params:
cmd.extend(shlex.split(self.extra_params))
if self.coverage:
build.coverage = True
coverage_extra_params = self._coverage_params(build, modules_to_install)
py_version = get_py_version(build)
cmd = [py_version, '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + coverage_extra_params + cmd
cmd += self._post_install_command(build, modules_to_install) # coverage post, extra-checks, ...
max_timeout = int(self.env['ir.config_parameter'].get_param('runbot.runbot_timeout', default=10000))
timeout = min(self.cpu_limit, max_timeout)
return docker_run(build_odoo_cmd(cmd), log_path, build._path(), build._get_docker_name(), cpu_limit=timeout)
def _modules_to_install(self, build):
modules_to_install = set([mod.strip() for mod in self.install_modules.split(',')])
if '*' in modules_to_install:
modules_to_install.remove('*')
default_mod = set([mod.strip() for mod in build.modules.split(',')])
modules_to_install = default_mod | modules_to_install
# todo add without support
return modules_to_install
def _post_install_command(self, build, modules_to_install):
if self.coverage:
py_version = get_py_version(build)
# prepare coverage result
cov_path = build._path('coverage')
os.makedirs(cov_path, exist_ok=True)
return ['&&', py_version, "-m", "coverage", "html", "-d", "/data/build/coverage", "--ignore-errors"]
return []
def _coverage_params(self, build, modules_to_install):
available_modules = [ # todo extract this to build method
os.path.basename(os.path.dirname(a))
for a in (glob.glob(build._server('addons/*/__openerp__.py')) +
glob.glob(build._server('addons/*/__manifest__.py')))
]
module_to_omit = set(available_modules) - modules_to_install
return ['--omit', ','.join('*addons/%s/*' % m for m in module_to_omit) + '*__manifest__.py']
def _make_results(self, build):
build_values = {}
if self.job_type == 'install_odoo':
if self.coverage:
build_values.update(self._make_coverage_results(build))
if self.test_enable or self.test_tags:
build_values.update(self._make_tests_results(build))
return build_values
def _make_coverage_results(self, build):
build_values = {}
build._log('coverage_result', 'Start getting coverage result')
cov_path = build._path('coverage/index.html')
if os.path.exists(cov_path):
with open(cov_path, 'r') as f:
data = f.read()
covgrep = re.search(r'pc_cov.>(?P<coverage>\d+)%', data)
build_values['coverage_result'] = covgrep and covgrep.group('coverage') or False
if build_values['coverage_result']:
build._log('coverage_result', 'Coverage result: %s' % build_values['coverage_result'])
else:
build._log('coverage_result', 'Coverage result not found', level='WARNING')
else:
build._log('coverage_result', 'Coverage file not found', level='WARNING')
return build_values
def _make_tests_results(self, build):
build_values = {}
if self.test_enable or self.test_tags:
build._log('run', 'Getting results for build %s' % build.dest)
log_file = build._path('logs', '%s.txt' % build.active_step.name)
if not os.path.isfile(log_file):
build_values['local_result'] = 'ko'
build._log('_checkout', "Log file not found at the end of test job", level="ERROR")
else:
log_time = time.localtime(os.path.getmtime(log_file))
build_values['job_end'] = time2str(log_time),
if not build.local_result or build.local_result in ['ok', "warn"]:
if grep(log_file, ".modules.loading: Modules loaded."):
local_result = False
if rfind(log_file, _re_error):
local_result = 'ko'
build._log('_checkout', 'Error or traceback found in logs', level="ERROR")
elif rfind(log_file, _re_warning):
local_result = 'warn'
build._log('_checkout', 'Warning found in logs', level="WARNING")
elif not grep(log_file, "Initiating shutdown"):
local_result = 'ko'
build._log('_checkout', 'No "Initiating shutdown" found in logs, maybe because of cpu limit.', level="ERROR")
else:
local_result = 'ok'
build_values['local_result'] = build._get_worst_result([build.local_result, local_result])
else:
build_values['local_result'] = 'ko'
build._log('_checkout', "Module loaded not found in logs", level="ERROR")
return build_values
def _step_state(self):
self.ensure_one()
if self.job_type == 'run_odoo' or (self.job_type == 'python' and self.running_job):
return 'running'
return 'testing'
def _has_log(self):
self.ensure_one()
return self.job_type != 'create_build'
class ConfigStepOrder(models.Model):
_name = 'runbot.build.config.step.order'
_order = 'sequence, id'
# a kind of many2many rel with sequence
sequence = fields.Integer('Sequence', required=True)
config_id = fields.Many2one('runbot.build.config', 'Config', required=True, ondelete='cascade')
step_id = fields.Many2one('runbot.build.config.step', 'Config Step', required=True, ondelete='cascade')
@api.onchange('step_id')
def _onchange_step_id(self):
self.sequence = self.step_id.default_sequence
@api.model
def create(self, values):
if 'sequence' not in values and values.get('step_id'):
values['sequence'] = self.env['runbot.build.config.step'].browse(values.get('step_id')).default_sequence
if self.pool._init: # do not duplicate entry on install
existing = self.search([('sequence', '=', values.get('sequence')), ('config_id', '=', values.get('config_id')), ('step_id', '=', values.get('step_id'))])
if existing:
return
return super(ConfigStepOrder, self).create(values)

1
runbot/models/build_dependency.py
View File

@ -1,5 +1,6 @@
from odoo import models, fields
class RunbotBuildDependency(models.Model):
_name = "runbot.build.dependency"

37
runbot/models/event.py
View File

@ -6,7 +6,7 @@ from odoo import models, fields, api
_logger = logging.getLogger(__name__)
TYPES = [(t, t.capitalize()) for t in 'client server runbot'.split()]
TYPES = [(t, t.capitalize()) for t in 'client server runbot subbuild'.split()]
class runbot_event(models.Model):
@ -24,25 +24,26 @@ class runbot_event(models.Model):
parent_class.init()
self._cr.execute("""
CREATE OR REPLACE FUNCTION runbot_set_logging_build() RETURNS TRIGGER AS $$
CREATE OR REPLACE FUNCTION runbot_set_logging_build() RETURNS TRIGGER AS $runbot_set_logging_build$
BEGIN
IF (new.build_id IS NULL AND new.dbname IS NOT NULL AND new.dbname != current_database()) THEN
UPDATE ir_logging l
SET build_id = split_part(new.dbname, '-', 1)::integer
WHERE l.id = new.id;
IF (NEW.build_id IS NULL AND NEW.dbname IS NOT NULL AND NEW.dbname != current_database()) THEN
NEW.build_id := split_part(NEW.dbname, '-', 1)::integer;
END IF;
RETURN NULL;
IF (NEW.build_id IS NOT NULL AND UPPER(NEW.level) NOT IN ('INFO', 'SEPARATOR')) THEN
BEGIN
UPDATE runbot_build b
SET triggered_result = CASE WHEN UPPER(NEW.level) = 'WARNING' THEN 'warn'
ELSE 'ko'
END
WHERE b.id = NEW.build_id;
END;
END IF;
RETURN NEW;
END;
$$ language plpgsql;
$runbot_set_logging_build$ language plpgsql;
DROP TRIGGER IF EXISTS runbot_new_logging ON ir_logging;
CREATE TRIGGER runbot_new_logging BEFORE INSERT ON ir_logging
FOR EACH ROW EXECUTE PROCEDURE runbot_set_logging_build();
DO $$
BEGIN
CREATE TRIGGER runbot_new_logging
AFTER INSERT ON ir_logging
FOR EACH ROW
EXECUTE PROCEDURE runbot_set_logging_build();
EXCEPTION
WHEN duplicate_object THEN
END;
$$;
""")

12
runbot/models/ir_cron.py Normal file
View File

@ -0,0 +1,12 @@
import odoo
from dateutil.relativedelta import relativedelta
from odoo import models, fields
odoo.service.server.SLEEP_INTERVAL = 5
odoo.addons.base.ir.ir_cron._intervalTypes['seconds'] = lambda interval: relativedelta(seconds=interval)
class ir_cron(models.Model):
_inherit = "ir.cron"
interval_type = fields.Selection(selection_add=[('seconds', 'Seconds')])

109
runbot/models/repo.py
View File

@ -40,7 +40,7 @@ class runbot_repo(models.Model):
modules_auto = fields.Selection([('none', 'None (only explicit modules list)'),
('repo', 'Repository modules (excluding dependencies)'),
('all', 'All modules (including dependencies)')],
default='repo',
default='all',
string="Other modules to install automatically")
dependency_ids = fields.Many2many(
@ -50,6 +50,20 @@ class runbot_repo(models.Model):
token = fields.Char("Github token", groups="runbot.group_runbot_admin")
group_ids = fields.Many2many('res.groups', string='Limited to groups')
repo_config_id = fields.Many2one('runbot.build.config', 'Run Config')
config_id = fields.Many2one('runbot.build.config', 'Run Config', compute='_compute_config_id', inverse='_inverse_config_id')
def _compute_config_id(self):
for repo in self:
if repo.repo_config_id:
repo.config_id = repo.repo_config_id
else:
repo.config_id = self.env.ref('runbot.runbot_build_config_default')
def _inverse_config_id(self):
for repo in self:
repo.repo_config_id = repo.config_id
def _root(self):
"""Return root directory of repository"""
default = os.path.join(os.path.dirname(__file__), '../static')
@ -83,7 +97,7 @@ class runbot_repo(models.Model):
"""Execute a git command 'cmd'"""
for repo in self:
cmd = ['git', '--git-dir=%s' % repo.path] + cmd
_logger.info("git command: %s", ' '.join(cmd))
_logger.debug("git command: %s", ' '.join(cmd))
return subprocess.check_output(cmd).decode('utf-8')
def _git_rev_parse(self, branch_name):
@ -197,6 +211,8 @@ class runbot_repo(models.Model):
# create build (and mark previous builds as skipped) if not found
if not (branch.id, sha) in builds_candidates:
if branch.no_auto_build or branch.no_build:
continue
_logger.debug('repo %s branch %s new build found revno %s', self.name, branch.name, sha)
build_info = {
'branch_id': branch.id,
@ -207,12 +223,11 @@ class runbot_repo(models.Model):
'committer_email': committer_email,
'subject': subject,
'date': dateutil.parser.parse(date[:19]),
'coverage': branch.coverage,
}
if not branch.sticky:
# pending builds are skipped as we have a new ref
builds_to_skip = Build.search(
[('branch_id', '=', branch.id), ('state', '=', 'pending')],
[('branch_id', '=', branch.id), ('local_state', '=', 'pending')],
order='sequence asc')
builds_to_skip._skip(reason='New ref found')
if builds_to_skip:
@ -220,12 +235,12 @@ class runbot_repo(models.Model):
# testing builds are killed
builds_to_kill = Build.search([
('branch_id', '=', branch.id),
('state', '=', 'testing'),
('local_state', '=', 'testing'),
('committer', '=', committer)
])
builds_to_kill.write({'state': 'deathrow'})
for btk in builds_to_kill:
btk._log('repo._update_git', 'Build automatically killed, newer build found.')
btk._log('repo._update_git', 'Build automatically killed, newer build found.', level='WARNING')
builds_to_kill.write({'local_state': 'deathrow'})
new_build = Build.create(build_info)
# create a reverse dependency build if needed
@ -239,7 +254,7 @@ class runbot_repo(models.Model):
new_build.revdep_build_ids += latest_rev_build._force(message='Rebuild from dependency %s commit %s' % (self.name, sha[:6]))
# skip old builds (if their sequence number is too low, they will not ever be built)
skippable_domain = [('repo_id', '=', self.id), ('state', '=', 'pending')]
skippable_domain = [('repo_id', '=', self.id), ('local_state', '=', 'pending')]
icp = self.env['ir.config_parameter']
running_max = int(icp.get_param('runbot.runbot_running_max', default=75))
builds_to_be_skipped = Build.search(skippable_domain, order='sequence desc', offset=running_max)
@ -286,11 +301,12 @@ class runbot_repo(models.Model):
fname_fetch_head = os.path.join(repo.path, 'FETCH_HEAD')
if not force and os.path.isfile(fname_fetch_head):
fetch_time = os.path.getmtime(fname_fetch_head)
if repo.mode == 'hook' and repo.hook_time and dt2time(repo.hook_time) < fetch_time:
if repo.mode == 'hook' and (not repo.hook_time or dt2time(repo.hook_time) < fetch_time):
t0 = time.time()
_logger.debug('repo %s skip hook fetch fetch_time: %ss ago hook_time: %ss ago',
repo.name, int(t0 - fetch_time), int(t0 - dt2time(repo.hook_time)))
repo.name, int(t0 - fetch_time), int(t0 - dt2time(repo.hook_time)) if repo.hook_time else 'never')
return
self._update_fetch_cmd()
def _update_fetch_cmd(self):
@ -320,52 +336,55 @@ class runbot_repo(models.Model):
host = fqdn()
Build = self.env['runbot.build']
domain = [('repo_id', 'in', ids), ('branch_id.job_type', '!=', 'none')]
domain = [('repo_id', 'in', ids)]
domain_host = domain + [('host', '=', host)]
# schedule jobs (transitions testing -> running, kill jobs, ...)
build_ids = Build.search(domain_host + [('state', 'in', ['testing', 'running', 'deathrow'])])
build_ids = Build.search(domain_host + [('local_state', 'in', ['testing', 'running', 'deathrow'])])
build_ids._schedule()
# launch new tests
nb_testing = Build.search_count(domain_host + [('state', '=', 'testing')])
available_slots = workers - nb_testing
if available_slots > 0:
# commit transaction to reduce the critical section duration
self.env.cr.commit()
# self-assign to be sure that another runbot instance cannot self assign the same builds
query = """UPDATE
runbot_build
SET
host = %(host)s
WHERE
runbot_build.id IN (
SELECT
runbot_build.id
FROM
runbot_build
LEFT JOIN runbot_branch ON runbot_branch.id = runbot_build.branch_id
WHERE
runbot_build.repo_id IN %(repo_ids)s
AND runbot_build.state = 'pending'
AND runbot_branch.job_type != 'none'
AND runbot_build.host IS NULL
ORDER BY
runbot_branch.sticky DESC,
runbot_branch.priority DESC,
array_position(array['normal','rebuild','indirect','scheduled']::varchar[], runbot_build.build_type) ASC,
runbot_build.sequence ASC
FOR UPDATE OF runbot_build SKIP LOCKED
LIMIT %(available_slots)s)"""
self.env.cr.execute(query, {'repo_ids': tuple(ids), 'host': fqdn(), 'available_slots': available_slots})
pending_build = Build.search(domain + domain_host + [('state', '=', 'pending')])
nb_testing = Build.search_count(domain_host + [('local_state', '=', 'testing')])
available_slots = workers - nb_testing
reserved_slots = Build.search_count(domain_host + [('local_state', '=', 'pending')])
assignable_slots = available_slots - reserved_slots
if available_slots > 0:
if assignable_slots > 0: # note: slots have been addapt to be able to force host on pending build. Normally there is no pending with host.
# commit transaction to reduce the critical section duration
self.env.cr.commit()
# self-assign to be sure that another runbot instance cannot self assign the same builds
query = """UPDATE
runbot_build
SET
host = %(host)s
WHERE
runbot_build.id IN (
SELECT
runbot_build.id
FROM
runbot_build
LEFT JOIN runbot_branch ON runbot_branch.id = runbot_build.branch_id
WHERE
runbot_build.repo_id IN %(repo_ids)s
AND runbot_build.local_state = 'pending'
AND runbot_build.host IS NULL
ORDER BY
runbot_branch.sticky DESC,
runbot_branch.priority DESC,
array_position(array['normal','rebuild','indirect','scheduled']::varchar[], runbot_build.build_type) ASC,
runbot_build.sequence ASC
FOR UPDATE OF runbot_build SKIP LOCKED
LIMIT %(assignable_slots)s)"""
self.env.cr.execute(query, {'repo_ids': tuple(ids), 'host': fqdn(), 'assignable_slots': assignable_slots})
pending_build = Build.search(domain_host + [('local_state', '=', 'pending')], limit=available_slots)
if pending_build:
pending_build._schedule()
self.env.cr.commit()
# terminate and reap doomed build
build_ids = Build.search(domain_host + [('state', '=', 'running')]).ids
build_ids = Build.search(domain_host + [('local_state', '=', 'running')]).ids
# sort builds: the last build of each sticky branch then the rest
sticky = {}
non_sticky = []
@ -393,7 +412,7 @@ class runbot_repo(models.Model):
settings['fqdn'] = fqdn()
nginx_repos = self.search([('nginx', '=', True)], order='id')
if nginx_repos:
settings['builds'] = self.env['runbot.build'].search([('repo_id', 'in', nginx_repos.ids), ('state', '=', 'running'), ('host', '=', fqdn())])
settings['builds'] = self.env['runbot.build'].search([('repo_id', 'in', nginx_repos.ids), ('local_state', '=', 'running'), ('host', '=', fqdn())])
nginx_config = self.env['ir.ui.view'].render_template("runbot.nginx_config", settings)
os.makedirs(nginx_dir, exist_ok=True)

4
runbot/models/res_config_settings.py
View File

@ -9,7 +9,7 @@ class ResConfigSettings(models.TransientModel):
runbot_workers = fields.Integer('Total number of workers')
runbot_running_max = fields.Integer('Maximum number of running builds')
runbot_timeout = fields.Integer('Default timeout (in seconds)')
runbot_timeout = fields.Integer('Max allowed step timeout (in seconds)')
runbot_starting_port = fields.Integer('Starting port for running builds')
runbot_domain = fields.Char('Runbot domain')
runbot_max_age = fields.Integer('Max branch age (in days)')
@ -22,7 +22,7 @@ class ResConfigSettings(models.TransientModel):
get_param = self.env['ir.config_parameter'].sudo().get_param
res.update(runbot_workers=int(get_param('runbot.runbot_workers', default=6)),
runbot_running_max=int(get_param('runbot.runbot_running_max', default=75)),
runbot_timeout=int(get_param('runbot.runbot_timeout', default=3600)),
runbot_timeout=int(get_param('runbot.runbot_timeout', default=10000)),
runbot_starting_port=int(get_param('runbot.runbot_starting_port', default=2000)),
runbot_domain=get_param('runbot.runbot_domain', default=common.fqdn()),
runbot_max_age=int(get_param('runbot.runbot_max_age', default=30)),

9
runbot/security/ir.model.access.csv
View File

@ -8,3 +8,12 @@ access_runbot_branch_admin,runbot_branch_admin,runbot.model_runbot_branch,runbot
access_runbot_build_admin,runbot_build_admin,runbot.model_runbot_build,runbot.group_runbot_admin,1,1,1,1
access_runbot_build_dependency_admin,runbot_build_dependency_admin,runbot.model_runbot_build_dependency,runbot.group_runbot_admin,1,1,1,1
access_irlogging,log by runbot users,base.model_ir_logging,group_user,0,0,1,0
access_runbot_build_config_step_user,runbot_build_config_step_user,runbot.model_runbot_build_config_step,group_user,1,0,0,0
access_runbot_build_config_step_manager,runbot_build_config_step_manager,runbot.model_runbot_build_config_step,runbot.group_build_config_user,1,1,1,1
access_runbot_build_config_user,runbot_build_config_user,runbot.model_runbot_build_config,group_user,1,0,0,0
access_runbot_build_config_manager,runbot_build_config_manager,runbot.model_runbot_build_config,runbot.group_build_config_user,1,1,1,1
access_runbot_build_config_step_order_user,runbot_build_config_step_order_user,runbot.model_runbot_build_config_step_order,group_user,1,0,0,0
access_runbot_build_config_step_order_manager,runbot_build_config_step_order_manager,runbot.model_runbot_build_config_step_order,runbot.group_build_config_user,1,1,1,1

1 id name model_id:id group_id:id perm_read perm_write perm_create perm_unlink
8 access_runbot_build_admin runbot_build_admin runbot.model_runbot_build runbot.group_runbot_admin 1 1 1 1
9 access_runbot_build_dependency_admin runbot_build_dependency_admin runbot.model_runbot_build_dependency runbot.group_runbot_admin 1 1 1 1
10 access_irlogging log by runbot users base.model_ir_logging group_user 0 0 1 0
11 access_runbot_build_config_step_user runbot_build_config_step_user runbot.model_runbot_build_config_step group_user 1 0 0 0
12 access_runbot_build_config_step_manager runbot_build_config_step_manager runbot.model_runbot_build_config_step runbot.group_build_config_user 1 1 1 1
13 access_runbot_build_config_user runbot_build_config_user runbot.model_runbot_build_config group_user 1 0 0 0
14 access_runbot_build_config_manager runbot_build_config_manager runbot.model_runbot_build_config runbot.group_build_config_user 1 1 1 1
15 access_runbot_build_config_step_order_user runbot_build_config_step_order_user runbot.model_runbot_build_config_step_order group_user 1 0 0 0
16 access_runbot_build_config_step_order_manager runbot_build_config_step_order_manager runbot.model_runbot_build_config_step_order runbot.group_build_config_user 1 1 1 1
17
18
19

86
runbot/security/runbot_security.xml
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<openerp>
<odoo>
<data>
<record model="ir.module.category" id="module_category">
<field name="name">Runbot</field>
@ -24,5 +24,87 @@
<field name="implied_ids" eval="[(4, ref('runbot.group_user'))]"/>
</record>
<record model="ir.module.category" id="build_config_category">
<field name="name">Build Config</field>
</record>
<record id="group_build_config_user" model="res.groups">
<field name="name">Build config user</field>
<field name="category_id" ref="build_config_category"/>
</record>
<record id="group_build_config_manager" model="res.groups">
<field name="name">Build config manager</field>
<field name="category_id" ref="build_config_category"/>
<field name="implied_ids" eval="[(4, ref('runbot.group_build_config_user'))]"/>
</record>
<record id="group_build_config_administrator" model="res.groups">
<field name="name">Build config administrator</field>
<field name="category_id" ref="build_config_category"/>
<field name="implied_ids" eval="[(4, ref('runbot.group_build_config_manager'))]"/>
<field name="users" eval="[(4, ref('base.user_root'))]"/>
</record>
<!-- config access rules-->
<record id="runbot_build_config_access_administrator" model="ir.rule">
<field name="name">All config can be edited by config admin</field>
<field name="groups" eval="[(4, ref('group_build_config_administrator'))]"/>
<field name="model_id" ref="model_runbot_build_config"/>
<field name="domain_force">[('1', '=', '1')]</field>
<field name="perm_write" eval="True"/>
<field name="perm_unlink" eval="True"/>
<field name="perm_read" eval="False"/>
<field name="perm_create" eval="False"/>
</record>
<record id="runbot_build_config_access_manager" model="ir.rule">
<field name="name">Own config can be edited by user</field>
<field name="groups" eval="[(4, ref('group_build_config_manager'))]"/>
<field name="model_id" ref="model_runbot_build_config"/>
<field name="domain_force">[('protected', '=', False)]</field>
<field name="perm_write" eval="True"/>
<field name="perm_unlink" eval="True"/>
<field name="perm_read" eval="False"/>
<field name="perm_create" eval="True"/>
</record>
<record id="runbot_build_config_access_user" model="ir.rule">
<field name="name">All config can be edited by config admin</field>
<field name="groups" eval="[(4, ref('group_build_config_user'))]"/>
<field name="model_id" ref="model_runbot_build_config"/>
<field name="domain_force">[('create_uid', '=', user.id)]</field>
<field name="perm_write" eval="True"/>
<field name="perm_unlink" eval="True"/>
<field name="perm_read" eval="False"/>
<field name="perm_create" eval="True"/>
</record>
<!-- step access rules-->
<record id="runbot_build_config_step_access_administrator" model="ir.rule">
<field name="name">All config step can be edited by config admin</field>
<field name="groups" eval="[(4, ref('group_build_config_administrator'))]"/>
<field name="model_id" ref="model_runbot_build_config_step"/>
<field name="domain_force">[('1', '=', '1')]</field>
<field name="perm_read" eval="False"/>
</record>
<record id="runbot_build_config_step_access_manager" model="ir.rule">
<field name="name">Unprotected config step can be edited by manager</field>
<field name="groups" eval="[(4, ref('group_build_config_manager'))]"/>
<field name="model_id" ref="model_runbot_build_config_step"/>
<field name="domain_force">[('protected', '=', False)]</field>
<field name="perm_read" eval="False"/>
</record>
<record id="runbot_build_config_step_access_user" model="ir.rule">
<field name="name">Own config step can be edited by user</field>
<field name="groups" eval="[(4, ref('group_build_config_user'))]"/>
<field name="model_id" ref="model_runbot_build_config_step"/>
<field name="domain_force">[('protected', '=', False), ('create_uid', '=', user.id)]</field>
<field name="perm_read" eval="False"/>
</record>
</data>
</openerp>
</odoo>

4
runbot/static/src/less/runbot.less Normal file
View File

@ -0,0 +1,4 @@
.separator {
border-top: 2px solid #666;
font-weight: bold;
}

20
runbot/templates/branch.xml
View File

@ -27,21 +27,23 @@
</tr>
</thead>
<t t-foreach="builds" t-as="build">
<t t-if="build['state'] in ['running','done'] and build['result'] == 'ko'"><t t-set="rowclass">danger</t></t>
<t t-if="build['state'] in ['running','done'] and build['result'] == 'warn'"><t t-set="rowclass">warning</t></t>
<t t-if="build['state'] in ['running','done'] and build['result'] == 'ok'"><t t-set="rowclass">success</t></t>
<t t-if="build['state'] in ['running','done'] and build['result'] == 'skipped'"><t t-set="rowclass">default</t></t>
<t t-if="build['state'] in ['running','done'] and build['result'] in ['killed', 'manually_killed']"><t t-set="rowclass">killed</t></t>
<t t-if="build.global_state in ['running','done']">
<t t-if="build.global_result == 'ko'"><t t-set="rowclass">danger</t></t>
<t t-if="build.global_result == 'warn'"><t t-set="rowclass">warning</t></t>
<t t-if="build.global_result == 'ok'"><t t-set="rowclass">success</t></t>
<t t-if="build.global_result == 'skipped'"><t t-set="rowclass">default</t></t>
<t t-if="build.global_result in ['killed', 'manually_killed']"><t t-set="rowclass">killed</t></t>
</t>
<tr t-attf-class="{{rowclass}}">
<td><t t-esc="build.sequence" /></td>
<td><t t-esc="build.date" /></td>
<td><a t-attf-href="/runbot/build/{{build['id']}}" title="Build details" aria-label="Build details"><t t-esc="build.dest" /></a></td>
<td><t t-esc="build.subject" /></td>
<td><t t-esc="build.name" /></td>
<td><t t-esc="build.result" /></td>
<td><t t-esc="build.state" /></td>
<td><t t-esc="build.host" /></td>
<td><t t-esc="build.job" /></td>
<td><t t-esc="build.global_result" /></td>
<td><t t-esc="build.global_state" /></td>
<td><t t-esc="build.real_build.host" /></td>
<td><t t-esc="build.active_step.name" /></td>
<td><t t-esc="build.job_time" /></td>
<td><t t-esc="build.build_type" /></td>
</tr>

166
runbot/templates/build.xml
View File

@ -2,77 +2,90 @@
<odoo>
<data>
<template id="runbot.build_name">
<t t-if="bu['state']=='deathrow'"><i class="text-info fa fa-crosshairs"/> killing</t>
<t t-if="bu['state']=='pending'"><i class="text-default fa fa-pause"/> pending</t>
<t t-if="bu['state']=='testing'">
<t t-set="textklass" t-value="dict(ko='danger', warn='warning').get(bu['guess_result'], 'info')"/>
<span t-attf-class="text-{{textklass}}"><i class="fa fa-spinner fa-spin"/> testing</span> <t t-esc="bu['job']"/> <small t-if="not hide_time"><t t-esc="bu['job_time']"/></small>
<t t-if="bu.global_state=='deathrow'"><i class="text-info fa fa-crosshairs"/> killing</t>
<t t-if="bu.global_state=='pending'"><i class="text-default fa fa-pause"/> pending</t>
<t t-if="bu.global_state in ('testing', 'waiting')">
<t t-set="textklass" t-value="dict(ko='danger', warn='warning').get(bu.global_result, 'info')"/>
<t t-if="bu.global_state == 'waiting'">
<span t-attf-class="text-{{textklass}}"><i class="fa fa-spinner fa-spin"/> <t t-esc="bu.global_state"/></span> <small t-if="not hide_time">time: <t t-esc="bu.get_formated_build_time()"/></small>
</t>
<t t-else="">
<span t-attf-class="text-{{textklass}}"><i class="fa fa-spinner fa-spin"/> <t t-esc="bu.global_state"/></span> <small> step <t t-esc="bu['job']"/>: </small><small t-if="not hide_time"><t t-esc="bu.get_formated_job_time()"/> -- time: <t t-esc="bu.get_formated_build_time()"/></small>
</t>
</t>
<t t-if="bu['result']=='ok'"><i class="text-success fa fa-thumbs-up" title="Success" aria-label="Success"/><small t-if="not hide_time"> age <t t-esc="bu['job_age']"/> time <t t-esc="bu['job_time']"/></small></t>
<t t-if="bu['result']=='ko'"><i class="text-danger fa fa-thumbs-down" title="Failed" aria-label="Failed"/><small t-if="not hide_time"> age <t t-esc="bu['job_age']"/> time <t t-esc="bu['job_time']"/></small></t>
<t t-if="bu['result']=='warn'"><i class="text-warning fa fa-warning" title="Some warnings" aria-label="Some warnings"/><small t-if="not hide_time"> age <t t-esc="bu['job_age']"/> time <t t-esc="bu['job_time']"/></small></t>
<t t-if="bu['result']=='skipped'"><i class="text-danger fa fa-ban"/> skipped</t>
<t t-if="bu['result']=='killed'"><i class="text-danger fa fa-times"/> killed</t>
<t t-if="bu['result']=='manually_killed'"><i class="text-danger fa fa-times"/> manually killed</t>
<t t-if="bu['server_match'] == 'default'">
<t t-if="bu.global_state in ('running', 'done')">
<t t-if="bu.global_result=='ok'"><i class="text-success fa fa-thumbs-up" title="Success" aria-label="Success"/><small t-if="not hide_time"> age: <t t-esc="bu.get_formated_build_age()"/> -- time: <t t-esc="bu.get_formated_build_time()"/></small></t>
<t t-if="bu.global_result=='ko'"><i class="text-danger fa fa-thumbs-down" title="Failed" aria-label="Failed"/><small t-if="not hide_time"> age: <t t-esc="bu.get_formated_build_age()"/> -- time: <t t-esc="bu.get_formated_build_time()"/></small></t>
<t t-if="bu.global_result=='warn'"><i class="text-warning fa fa-warning" title="Some warnings" aria-label="Some warnings"/><small t-if="not hide_time"> age: <t t-esc="bu.get_formated_build_age()"/> -- time: <t t-esc="bu.get_formated_build_time()"/></small></t>
<t t-if="bu.global_result=='skipped'"><i class="text-danger fa fa-ban"/> skipped</t>
<t t-if="bu.global_result=='killed'"><i class="text-danger fa fa-times"/> killed</t>
<t t-if="bu.global_result=='manually_killed'"><i class="text-danger fa fa-times"/> manually killed</t>
</t>
<t t-if="bu.real_build.server_match == 'default'">
<i class="text-warning fa fa-question-circle fa-fw"
title="Server branch cannot be determined exactly. Please use naming convention '12.0-my-branch' to build with '12.0' server branch."/>
</t>
<t t-if="bu['revdep_build_ids']">
<t t-if="bu.revdep_build_ids">
<small class="pull-right">Dep builds:
<t t-foreach="bu['revdep_build_ids']" t-as="rbu">
<t t-foreach="bu.sorted_revdep_build_ids()" t-as="rbu">
<a t-attf-href="/runbot/build/{{rbu['id']}}" aria-label="Build details" data-toggle="tooltip" t-attf-title="{{rbu['repo_id']['name']}}">
<t t-if="rbu['result']=='ok'"><i class="text-success fa fa-thumbs-up"/></t>
<t t-if="rbu['result']=='ko'"><i class="text-danger fa fa-thumbs-down"/></t>
<t t-if="rbu['result']=='warn'"><i class="text-warning fa fa-warning"/></t>
<t t-if="rbu['result']=='skipped'"><i class="text-danger fa fa-ban"/></t>
<t t-if="rbu['result']=='killed'"><i class="text-danger fa fa-times"/>&#x1f480;</t>
<t t-if="rbu['result']=='manually_killed'"><i class="text-danger fa fa-times"/>&#x1f52b;</t>
<t t-if="rbu['state']=='deathrow'"><i class="fa fa-crosshairs" style="color: #666666;"/></t>
<t t-if="rbu['state']=='pending'"><i class="fa fa-pause" style="color: #666666;"/></t>
<t t-if="rbu['state']=='testing'"><i class="fa fa-spinner fa-spin" style="color: #666666;"/></t>
<t t-if="rbu.global_result=='ok'"><i class="text-success fa fa-thumbs-up"/></t>
<t t-if="rbu.global_result=='ko'"><i class="text-danger fa fa-thumbs-down"/></t>
<t t-if="rbu.global_result=='warn'"><i class="text-warning fa fa-warning"/></t>
<t t-if="rbu.global_result=='skipped'"><i class="text-danger fa fa-ban"/></t>
<t t-if="rbu.global_result=='killed'"><i class="text-danger fa fa-times"/>&#x1f480;</t>
<t t-if="rbu.global_result=='manually_killed'"><i class="text-danger fa fa-times"/>&#x1f52b;</t>
<t t-if="rbu.global_state=='deathrow'"><i class="fa fa-crosshairs" style="color: #666666;"/></t>
<t t-if="rbu.global_state=='pending'"><i class="fa fa-pause" style="color: #666666;"/></t>
<t t-if="rbu.global_state in ('testing', 'waiting')"><i class="fa fa-spinner fa-spin" style="color: #666666;"/></t>
</a>
</t>
</small>
</t>
<t t-set="nb_sum" t-value="bu.nb_pending+bu.nb_testing+bu.nb_running"/>
<t t-if="nb_sum > 1"><!-- maybe only display this info if > 3 -->
<span t-attf-title="{{bu.nb_pending}} pending, {{bu.nb_testing}} testing, {{bu.nb_running}} running">
<t t-esc="nb_sum"/>
<i class="fa fa-cogs"/>
</span>
</t>
</template>
<template id="runbot.build_button">
<div t-attf-class="pull-right">
<div t-attf-class="btn-group {{klass}}">
<a t-if="bu['state']=='running'" t-attf-href="http://{{bu['domain']}}/?db={{bu['real_dest']}}-all" class="btn btn-primary" title="Sign in on this build" aria-label="Sign in on this build"><i class="fa fa-sign-in"/></a>
<a t-if="bu.global_state=='running'" t-attf-href="http://{{bu['domain']}}/?db={{bu['real_build'].dest}}-all" class="btn btn-primary" title="Sign in on this build" aria-label="Sign in on this build"><i class="fa fa-sign-in"/></a>
<a t-attf-href="/runbot/build/{{bu['id']}}" class="btn btn-default" title="Build details" aria-label="Build details"><i class="fa fa-file-text-o"/></a>
<a t-attf-href="https://#{repo.base}/commit/#{bu['name']}" class="btn btn-default" title="Open commit on GitHub" aria-label="Open commit on GitHub"><i class="fa fa-github"/></a>
<button class="btn btn-default dropdown-toggle" data-toggle="dropdown" title="Build options" aria-label="Build options" aria-expanded="false"><i class="fa fa-cog"/><span class="caret"></span></button>
<ul class="dropdown-menu" role="menu">
<li t-if="bu['result']=='skipped'" groups="runbot.group_runbot_admin">
<li t-if="bu.global_result=='skipped'" groups="runbot.group_runbot_admin">
<a href="#" class="runbot-rebuild" t-att-data-runbot-build="bu['id']">Force Build <i class="fa fa-level-up"></i></a>
</li>
<t t-if="bu['state']=='running'">
<li><a t-attf-href="http://{{bu['domain']}}/?db={{bu['real_dest']}}-all">Connect all <i class="fa fa-sign-in"></i></a></li>
<li><a t-attf-href="http://{{bu['domain']}}/?db={{bu['real_dest']}}-base">Connect base <i class="fa fa-sign-in"></i></a></li>
<t t-if="bu.global_state=='running'">
<li><a t-attf-href="http://{{bu['domain']}}/?db={{bu['real_build'].dest}}-all">Connect all <i class="fa fa-sign-in"></i></a></li>
<li><a t-attf-href="http://{{bu['domain']}}/?db={{bu['real_build'].dest}}-base">Connect base <i class="fa fa-sign-in"></i></a></li>
<li><a t-attf-href="http://{{bu['domain']}}/">Connect <i class="fa fa-sign-in"></i></a></li>
</t>
<li t-if="bu['state'] in ['done','running','deathrow'] and bu_index==0" groups="runbot.group_user">
<li t-if="bu.global_state in ['done','running','deathrow'] and bu_index==0" groups="runbot.group_user">
<a href="#" class="runbot-rebuild" t-att-data-runbot-build="bu['id']">Rebuild <i class="fa fa-refresh"/></a>
</li>
<li t-if="bu['state'] in ['pending','testing','running']" groups="runbot.group_user">
<li t-if="bu.global_state in ['pending','testing', 'waiting', 'running']" groups="runbot.group_user">
<a href="#" class="runbot-kill" t-att-data-runbot-build="bu['id']">Kill <i class="fa fa-crosshairs"/></a>
</li>
<li t-if="bu['state']!='testing' and bu['state']!='pending'" class="divider"></li>
<li t-if="bu.global_state not in ('testing', 'waiting', 'pending')" class="divider"></li>
<li><a t-attf-href="/runbot/build/{{bu['id']}}">Logs <i class="fa fa-file-text-o"/></a></li>
<li t-if="bu['host']"><a t-attf-href="http://{{bu['host']}}/runbot/static/build/#{bu['real_dest']}/logs/job_10_test_base.txt">Full base logs <i class="fa fa-file-text-o"/></a></li>
<li t-if="bu['host']"><a t-attf-href="http://{{bu['host']}}/runbot/static/build/#{bu['real_dest']}/logs/job_20_test_all.txt">Full all logs <i class="fa fa-file-text-o"/></a></li>
<li t-if="bu['coverage'] and bu['host']"><a t-attf-href="http://{{bu['host']}}/runbot/static/build/#{bu['real_dest']}/coverage/index.html">Coverage <i class="fa fa-file-text-o"/></a></li>
<li t-if="bu['state']!='pending'" class="divider"></li>
<t t-set="log_url" t-value="'http://%s' % bu.real_build.host if bu.real_build.host != fqdn else ''"/>
<t t-if="bu.real_build.host" t-foreach="bu.log_list.split(',')" t-as="log_name" >
<li><a t-attf-href="{{log_url}}/runbot/static/build/#{bu['real_build'].dest}/logs/#{log_name}.txt">Full <t t-esc="log_name"/> logs <i class="fa fa-file-text-o"/></a></li>
</t>
<li t-if="bu.coverage and bu.real_build.host"><a t-attf-href="http://{{bu.real_build.host}}/runbot/static/build/#{bu['real_build'].dest}/coverage/index.html">Coverage <i class="fa fa-file-text-o"/></a></li>
<li t-if="bu.global_state!='pending'" class="divider"></li>
<li><a t-attf-href="{{br['branch'].branch_url}}">Branch or pull <i class="fa fa-github"/></a></li>
<li><a t-attf-href="https://{{repo.base}}/commit/{{bu['name']}}">Commit <i class="fa fa-github"/></a></li>
<li><a t-attf-href="https://{{repo.base}}/compare/{{br['branch'].branch_name}}">Compare <i class="fa fa-github"/></a></li>
<!-- TODO branch.pull from -->
<li class="divider"></li>
<li class="disabled"><a href="#">Runtime: <t t-esc="bu['job_time']"/>s</a></li>
<li class="disabled"><a href="#">Port: <t t-esc="bu['port']"/></a></li>
<li class="disabled"><a href="#">Age: <t t-esc="bu['job_age']"/></a></li>
<li class="disabled"><a href="#">Port: <t t-esc="bu.real_build.port"/></a></li>
</ul>
</div>
</div>
@ -162,13 +175,61 @@
</div>
</div>
</nav>
<p>
Subject: <t t-esc="build['subject']"/><br/>
Author: <t t-esc="build['author']"/><br/>
Committer: <t t-esc="build['committer']"/><br/>
Build host: <t t-esc="build['host']"/><br/>
</p>
<p t-if="build['duplicate_of']">Duplicate of <a t-attf-href="/runbot/build/#{build['duplicate_of'].id}"><t t-esc="build['duplicate_of'].dest"/></a></p>
<t t-set="rowclass">info</t>
<t t-if="build.global_state in ['running','done']">
<t t-if="build.global_result == 'ko'"><t t-set="rowclass">danger</t></t>
<t t-if="build.global_result == 'warn'"><t t-set="rowclass">warning</t></t>
<t t-if="build.global_result == 'ok'"><t t-set="rowclass">success</t></t>
<t t-if="build.global_result == 'skipped'"><t t-set="rowclass">default</t></t>
<t t-if="build.global_result in ['killed', 'manually_killed']"><t t-set="rowclass">killed</t></t>
</t>
<table class="table table-condensed tabel-bordered">
<tr>
<td t-attf-class="{{rowclass}}">
Subject: <t t-esc="build['subject']"/><br/>
Author: <t t-esc="build['author']"/><br/>
Committer: <t t-esc="build['committer']"/><br/>
Commit: <a t-attf-href="https://{{build.repo_id.base}}/commit/{{build.name}}"><t t-esc="build.name"/></a><br/>
<t t-foreach="build.dependency_ids" t-as="dep">
Dep: <t t-esc="dep.dependecy_repo_id.short_name"/>:<a t-attf-href="https://{{dep.dependecy_repo_id.base}}/commit/{{dep.dependency_hash}}"><t t-esc="dep.dependency_hash"/></a>
<t t-if='dep.closest_branch_id'> from branch <t t-esc="dep.closest_branch_id.name"/></t>
<br/>
</t>
Build host: <t t-esc="build.real_build.host"/><br/>
</td>
<td t-if="build.children_ids">
Children:
<t t-if="build.nb_pending > 0"><t t-esc="build.nb_pending"/> pending </t>
<t t-if="build.nb_testing > 0"><t t-esc="build.nb_testing"/> testing </t>
<t t-if="build.nb_running > 0"><t t-esc="build.nb_running"/> running </t>
<table class="table table-condensed">
<t t-foreach="build.children_ids" t-as="child">
<t t-set="rowclass">info</t>
<t t-if="child.global_state in ['running','done']">
<t t-if="child.global_result == 'ko'"><t t-set="rowclass">danger</t></t>
<t t-if="child.global_result == 'warn'"><t t-set="rowclass">warning</t></t>
<t t-if="child.global_result == 'ok'"><t t-set="rowclass">success</t></t>
<t t-if="child.global_result == 'skipped'"><t t-set="rowclass">default</t></t>
<t t-if="child.global_result in ['killed', 'manually_killed']"><t t-set="rowclass">killed</t></t>
</t>
<tr><td t-attf-class="{{rowclass}}">
<a t-attf-href="/runbot/build/{{child.id}}" >Build <t t-esc="child.id"/></a>
with config <a t-attf-href="/web#id={{child.config_id.id}}&amp;view_type=form&amp;model=runbot.build.config"><t t-esc="child.config_id.name"/></a>
<t t-if="child.job"> Running step: <t t-esc="child.job"/></t>
<t t-if="child.global_state in ['testing', 'waiting']">
<i class="fa fa-spinner fa-spin"/>
<t t-esc="child.global_state"/>
</t>
<a t-if="child.global_state=='running'" t-attf-href="http://{{child.domain}}/?db={{child.real_build.dest}}-all" title="Sign in on this build" aria-label="Sign in on this build"><i class="fa fa-sign-in"/></a>
</td></tr>
</t>
</table>
</td>
</tr>
</table>
<p t-if="build.parent_id">Child of <a t-attf-href="/runbot/build/#{build.parent_id.id}"><t t-esc="build.parent_id.dest"/></a></p>
<p t-if="build.duplicate_id">Duplicate of <a t-attf-href="/runbot/build/#{build.duplicate_id.id}"><t t-esc="build.duplicate_id.dest"/></a></p>
<table class="table table-condensed table-striped">
<tr>
<th>Date</th>
@ -177,13 +238,16 @@
<th>Message</th>
</tr>
<t t-foreach="logs" t-as="l">
<tr t-att-class="dict(ERROR='danger', WARNING='warning').get(l.level)">
<tr t-att-class="dict(ERROR='danger', WARNING='warning', OK='success', SEPARATOR='separator').get(l.level)">
<td style="white-space: nowrap;"><t t-esc="l.create_date"/></td>
<td><b t-esc="l.level"/></td>
<td><t t-esc="l.type"/></td>
<td><b t-if="l.level != 'SEPARATOR'" t-esc="l.level"/></td>
<td><t t-if="l.level != 'SEPARATOR'" t-esc="l.type"/></td>
<td>
<a t-if="l.type != 'runbot'" t-attf-href="https://{{repo.base}}/blob/{{build['name']}}/{{l.path}}#L{{l.line}}"><t t-esc="l.name"/>:<t t-esc="l.line"/></a> <t t-esc="l.func"/>
<t t-if="'\n' not in l.message"><t t-esc="l.message"/></t>
<t t-if="l.type != 'runbot'">
<a t-if="l.type == 'subbuild'" t-attf-href="/runbot/build/{{l.path}}">Sub build #<t t-esc="l.path"/></a>
<a t-else="" t-attf-href="https://{{repo.base}}/blob/{{build['name']}}/{{l.path}}#L{{l.line}}"><t t-esc="l.name"/>:<t t-esc="l.line"/></a> <t t-esc="l.func"/>
</t>
<t t-if="'\n' not in l.message" t-esc="l.message"/>
<pre t-if="'\n' in l.message" style="margin:0;padding:0; border: none;"><t t-esc="l.message"/></pre>
</td>
</tr>

24
runbot/templates/dashboard.xml
View File

@ -41,18 +41,18 @@
<div t-foreach="repo['branches'].values()" t-as="br">
<div class="col-md-1">
<b t-esc="br['name']"/><br/>
<small><t t-esc="br['builds'][0]['job_age']"/></small>
<small><t t-esc="br['builds'][0].get_formated_build_time()"/></small>
</div>
<div class="col-md-11 r-mb02">
<t t-foreach="br['builds']" t-as="bu">
<t t-if="bu['state']=='pending'"><t t-set="klass">default</t></t>
<t t-if="bu['state']=='testing'"><t t-set="klass">info</t></t>
<t t-if="bu['state']=='deathrow'"><t t-set="klass">default</t></t>
<t t-if="bu['state'] in ['running','done'] and bu['result'] == 'ko'"><t t-set="klass">danger</t></t>
<t t-if="bu['state'] in ['running','done'] and bu['result'] == 'warn'"><t t-set="klass">warning</t></t>
<t t-if="bu['state'] in ['running','done'] and bu['result'] == 'ok'"><t t-set="klass">success</t></t>
<t t-if="bu['state'] in ['running','done'] and bu['result'] == 'skipped'"><t t-set="klass">default</t></t>
<t t-if="bu['state'] in ['running','done'] and bu['result'] in ['killed', 'manually_killed']"><t t-set="klass">killed</t></t>
<t t-if="bu.global_state=='pending'"><t t-set="klass">default</t></t>
<t t-if="bu.global_state in ('testing', 'waiting')"><t t-set="klass">info</t></t>
<t t-if="bu.global_state=='deathrow'"><t t-set="klass">default</t></t>
<t t-if="bu.global_state in ['running','done'] and bu.global_result == 'ko'"><t t-set="klass">danger</t></t>
<t t-if="bu.global_state in ['running','done'] and bu.global_result == 'warn'"><t t-set="klass">warning</t></t>
<t t-if="bu.global_state in ['running','done'] and bu.global_result == 'ok'"><t t-set="klass">success</t></t>
<t t-if="bu.global_state in ['running','done'] and bu.global_result == 'skipped'"><t t-set="klass">default</t></t>
<t t-if="bu.global_state in ['running','done'] and bu.global_result in ['killed', 'manually_killed']"><t t-set="klass">killed</t></t>
<div t-attf-class="bg-{{klass}} col-md-4">
<i class="fa fa-at"></i>
<t t-esc="bu['author']"/>
@ -61,11 +61,11 @@
</t>
<br/>
<i class="fa fa-envelope-o"></i>
<t t-if="bu['build_type']=='scheduled'"><i class="fa fa-moon-o" t-att-title="bu['build_type_label']" t-att-aria-label="bu['build_type_label']"/></t>
<t t-if="bu['build_type'] in ('rebuild', 'redirect')"><i class="fa fa-recycle" t-att-title="bu['build_type_label']" t-att-aria-label="bu['build_type_label']"/></t>
<t t-if="bu['build_type']=='scheduled'"><i class="fa fa-moon-o" t-att-title="bu.build_type_label()" t-att-aria-label="bu.build_type_label()"/></t>
<t t-if="bu['build_type'] in ('rebuild', 'redirect')"><i class="fa fa-recycle" t-att-title="bu.build_type_label()" t-att-aria-label="bu.build_type_label()"/></t>
<a t-attf-href="https://#{repo['base']}/commit/#{bu['name']}"><t t-esc="bu['subject'][:32] + ('...' if bu['subject'][32:] else '') " t-att-title="bu['subject']"/></a>
<br/>
<t t-call="runbot.build_name"/> — <small><a t-attf-href="/runbot/build/{{bu['id']}}"><t t-esc="bu['dest']"/></a> on <t t-esc="bu['host']"/> <a t-if="bu['state'] == 'running'" t-attf-href="http://{{bu['domain']}}/?db={{bu['dest']}}-all"><i class="fa fa-sign-in"></i></a></small>
<t t-call="runbot.build_name"/> — <small><a t-attf-href="/runbot/build/{{bu['id']}}"><t t-esc="bu['dest']"/></a> on <t t-esc="bu.real_build.host"/> <a t-if="bu.global_state == 'running'" t-attf-href="http://{{bu['domain']}}/?db={{bu['dest']}}-all"><i class="fa fa-sign-in"></i></a></small>
</div>
</t>
</div>

28
runbot/templates/frontend.xml
View File

@ -116,7 +116,7 @@
<td>
<i t-if="br['branch'].sticky" class="fa fa-star" style="color: #f0ad4e" />
<a t-attf-href="/runbot/branch/#{br['branch'].id}"><b t-esc="br['branch'].branch_name"/></a>
<small><t t-esc="br['builds'] and br['builds'][0]['job_age']"/></small><br/>
<small><t t-esc="br['builds'] and br['builds'][0].get_formated_build_time()"/></small><br/>
<div class="btn-group btn-group-xs">
<a t-attf-href="{{br['branch'].branch_url}}" class="btn btn-default btn-xs">Branch or pull <i class="fa fa-github"/></a>
<a t-attf-href="/runbot/#{repo.id}/#{br['branch'].branch_name}" class="btn btn-default btn-xs" aria-label="Quick Connect"><i class="fa fa-fast-forward" title="Quick Connect"/></a>
@ -125,7 +125,7 @@
<br/>
<t t-if="br['branch'].coverage_result > 0">
<t t-set="last_build" t-value="br['branch']._get_last_coverage_build()" />
<a t-attf-href="http://{{last_build['host']}}/runbot/static/build/#{last_build['dest']}/coverage/index.html">
<a t-attf-href="http://{{last_build.real_build.host}}/runbot/static/build/#{last_build['dest']}/coverage/index.html">
<span class="label label-info">cov: <t t-esc="br['branch'].coverage_result"/>%</span>
</a>
</t>
@ -135,18 +135,20 @@
</t>
</td>
<t t-foreach="br['builds']" t-as="bu">
<t t-if="bu['state']=='pending'"><t t-set="klass">default</t></t>
<t t-if="bu['state']=='testing'"><t t-set="klass">info</t></t>
<t t-if="bu['state']=='deathrow'"><t t-set="klass">default</t></t>
<t t-if="bu['state'] in ['running','done'] and bu['result'] == 'ko'"><t t-set="klass">danger</t></t>
<t t-if="bu['state'] in ['running','done'] and bu['result'] == 'warn'"><t t-set="klass">warning</t></t>
<t t-if="bu['state'] in ['running','done'] and bu['result'] == 'ok'"><t t-set="klass">success</t></t>
<t t-if="bu['state'] in ['running','done'] and bu['result'] == 'skipped'"><t t-set="klass">default</t></t>
<t t-if="bu['state'] in ['running','done'] and bu['result'] in ['killed', 'manually_killed']"><t t-set="klass">killed</t></t>
<t t-if="bu.global_state=='pending'"><t t-set="klass">default</t></t>
<t t-if="bu.global_state in ('testing', 'waiting')"><t t-set="klass">info</t></t>
<t t-if="bu.global_state=='deathrow'"><t t-set="klass">default</t></t>
<t t-if="bu.global_state in ['running','done']">
<t t-if="bu.global_result == 'ko'"><t t-set="klass">danger</t></t>
<t t-if="bu.global_result == 'warn'"><t t-set="klass">warning</t></t>
<t t-if="bu.global_result == 'ok'"><t t-set="klass">success</t></t>
<t t-if="bu.global_result == 'skipped'"><t t-set="klass">default</t></t>
<t t-if="bu.global_result in ['killed', 'manually_killed']"><t t-set="klass">killed</t></t>
</t>
<td t-attf-class="{{klass}}">
<t t-call="runbot.build_button"><t t-set="klass">btn-group-sm</t></t>
<t t-if="bu['build_type']=='scheduled'"><i class="fa fa-moon-o" t-att-title="bu['build_type_label']" t-att-aria-label="bu['build_type_label']"/></t>
<t t-if="bu['build_type'] in ('rebuild', 'indirect')"><i class="fa fa-recycle" t-att-title="bu['build_type_label']" t-att-aria-label="bu['build_type_label']"/></t>
<t t-if="bu['build_type']=='scheduled'"><i class="fa fa-moon-o" t-att-title="bu.build_type_label()" t-att-aria-label="bu.build_type_label()"/></t>
<t t-if="bu['build_type'] in ('rebuild', 'indirect')"><i class="fa fa-recycle" t-att-title="bu.build_type_label()" t-att-aria-label="bu.build_type_label()"/></t>
<t t-if="bu['subject']">
<span t-esc="bu['subject'][:32] + ('...' if bu['subject'][32:] else '') " t-att-title="bu['subject']"/>
<br/>
@ -158,7 +160,7 @@
</t>
<br/>
</t>
<small><t t-esc="bu['dest']"/> on <t t-esc="bu['host']"/></small><br/>
<small><t t-esc="bu['dest']"/> on <t t-esc="bu.real_build.host"/></small><br/>
<t t-call="runbot.build_name"/>
</td>
</t>

2
runbot/tests/__init__.py
View File

@ -1,8 +1,6 @@
from . import test_repo
from . import test_branch
from . import test_build
from . import test_jobs
from . import test_frontend
from . import test_job_types
from . import test_schedule
from . import test_cron

8
runbot/tests/test_branch.py
View File

@ -22,7 +22,7 @@ class Test_Branch(common.TransactionCase):
self.assertEqual(branch.branch_name, 'master')
self.assertEqual(branch.branch_url, 'https://example.com/foo/bar/tree/master')
self.assertEqual(branch.job_type, 'all')
self.assertEqual(branch.config_id, self.env.ref('runbot.runbot_build_config_default'))
@patch('odoo.addons.runbot.models.repo.runbot_repo._github')
def test_pull_request(self, mock_github):
@ -45,9 +45,9 @@ class Test_Branch(common.TransactionCase):
'repo_id': self.repo.id,
'name': 'refs/head/foo-branch-bar'
})
self.assertFalse(branch.coverage)
self.assertEqual(branch.config_id, self.env.ref('runbot.runbot_build_config_default'))
cov_branch = self.Branch.create({
'repo_id': self.repo.id,
'name': 'refs/head/foo-coverage-branch-bar'
'name': 'refs/head/foo-use-coverage-branch-bar'
})
self.assertTrue(cov_branch.coverage)
self.assertEqual(cov_branch.config_id, self.env.ref('runbot.runbot_build_config_test_coverage'))

93
runbot/tests/test_build.py
View File

@ -40,41 +40,12 @@ class Test_Build(common.TransactionCase):
# Test domain compute with fqdn and ir.config_parameter
mock_fqdn.return_value = 'runbot98.nowhere.org'
self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_domain', False)
self.assertEqual(build.domain, 'runbot98.nowhere.org:1234')
self.env['ir.config_parameter'].set_param('runbot.runbot_domain', 'runbot99.example.org')
build._get_domain()
build._compute_domain()
self.assertEqual(build.domain, 'runbot99.example.org:1234')
@patch('odoo.addons.runbot.models.build.fqdn')
def test_guess_result(self, mock_fqdn):
build = self.Build.create({
'branch_id': self.branch.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
'port': '1234',
})
# Testing the guess_result computed field
self.assertEqual(build.guess_result, '', 'A pending build guess_result should be empty')
build.write({'state': 'done', 'result': 'ko'})
build.invalidate_cache()
self.assertEqual(build.guess_result, 'ko', 'A finished build should return the same as result')
build.write({'state': 'testing'})
build.invalidate_cache()
self.assertEqual(build.guess_result, 'ok', 'A testing build without logs should be ok')
self.env.cr.execute("""
INSERT INTO ir_logging(name, type, path, func, line, build_id, level, message)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""", ('testing', 'server', 'somewhere', 'test', 0, build.id, 'WARNING', 'blabla'))
build.invalidate_cache()
self.assertEqual(build.guess_result, 'warn', 'A testing build with warnings should be warn')
self.env.cr.execute("""
INSERT INTO ir_logging(name, type, path, func, line, build_id, level, message)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""", ('testing', 'server', 'somewhere', 'test', 0, build.id, 'ERROR', 'blabla'))
build.invalidate_cache()
self.assertEqual(build.guess_result, 'ko', 'A testing build with errors should be ko')
@patch('odoo.addons.runbot.models.build.os.mkdir')
@patch('odoo.addons.runbot.models.build.grep')
def test_build_cmd_log_db(self, mock_grep, mock_mkdir):
@ -89,51 +60,41 @@ class Test_Build(common.TransactionCase):
cmd = build._cmd()[0]
self.assertIn('--log-db=%s' % uri, cmd)
def test_build_job_type_from_branch_default(self):
"""test build job_type is computed from branch default job_type"""
def test_build_config_from_branch_default(self):
"""test build config_id is computed from branch default config_id"""
build = self.Build.create({
'branch_id': self.branch.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
})
self.assertEqual(build.job_type, 'all', "job_type should be the same as the branch")
self.assertEqual(build.config_id, self.env.ref('runbot.runbot_build_config_default'))
def test_build_job_type_from_branch_testing(self):
"""test build job_type is computed from branch"""
self.branch.job_type = 'testing'
def test_build_config_from_branch_testing(self):
"""test build config_id is computed from branch"""
self.branch.config_id = self.env.ref('runbot.runbot_build_config_default_no_run')
build = self.Build.create({
'branch_id': self.branch.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
})
self.assertEqual(build.job_type, 'testing', "job_type should be the same as the branch")
self.assertEqual(build.config_id, self.branch.config_id, "config_id should be the same as the branch")
def test_build_job_type_from_branch_none(self):
"""test build is not even created when branch job_type is none"""
self.branch.job_type = 'none'
def test_build_from_branch_no_build(self):
"""test build is not even created when branch no_build is True"""
self.branch.no_build = True
build = self.Build.create({
'branch_id': self.branch.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
})
self.assertEqual(build, self.Build, "build should be an empty recordset")
def test_build_job_type_can_be_set(self):
"""test build job_type can be set to something different than the one on the branch"""
self.branch.job_type = 'running'
def test_build_config_can_be_set(self):
"""test build config_id can be set to something different than the one on the branch"""
self.branch.config_id = self.env.ref('runbot.runbot_build_config_default')
build = self.Build.create({
'branch_id': self.branch.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
'job_type': 'testing'
'config_id': self.env.ref('runbot.runbot_build_config_default_no_run').id
})
self.assertEqual(build.job_type, 'testing', "job_type should be the one set on the build")
def test_build_job_type_none(self):
"""test build job_type set to none does not create a build"""
self.branch.job_type = 'running'
build = self.Build.create({
'branch_id': self.branch.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
'job_type': 'none'
})
self.assertEqual(build, self.Build, "build should be an empty recordset")
self.assertEqual(build.config_id, self.env.ref('runbot.runbot_build_config_default_no_run'), "config_id should be the one set on the build")
@patch('odoo.addons.runbot.models.build._logger')
def test_build_skip(self, mock_logger):
@ -144,8 +105,8 @@ class Test_Build(common.TransactionCase):
'port': '1234',
})
build._skip()
self.assertEqual(build.state, 'done')
self.assertEqual(build.result, 'skipped')
self.assertEqual(build.local_state, 'done')
self.assertEqual(build.local_result, 'skipped')
other_build = self.Build.create({
'branch_id': self.branch.id,
@ -153,8 +114,8 @@ class Test_Build(common.TransactionCase):
'port': '1234',
})
other_build._skip(reason='A good reason')
self.assertEqual(other_build.state, 'done')
self.assertEqual(other_build.result, 'skipped')
self.assertEqual(other_build.local_state, 'done')
self.assertEqual(other_build.local_result, 'skipped')
log_first_part = '%s skip %%s' % (other_build.dest)
mock_logger.debug.assert_called_with(log_first_part, 'A good reason')
@ -170,12 +131,12 @@ class Test_Build(common.TransactionCase):
'branch_id': self.branch_10.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
})
build2.write({'state': 'duplicate', 'duplicate_id': build1.id}) # this may not be usefull if we detect duplicate in same repo.
build2.write({'local_state': 'duplicate', 'duplicate_id': build1.id}) # this may not be usefull if we detect duplicate in same repo.
self.assertEqual(build1.state, 'pending')
self.assertEqual(build1.local_state, 'pending')
build2._ask_kill()
self.assertEqual(build1.state, 'done', 'A killed pending duplicate build should mark the real build as done')
self.assertEqual(build1.result, 'skipped', 'A killed pending duplicate build should mark the real build as skipped')
self.assertEqual(build1.local_state, 'done', 'A killed pending duplicate build should mark the real build as done')
self.assertEqual(build1.local_result, 'skipped', 'A killed pending duplicate build should mark the real build as skipped')
def rev_parse(repo, branch_name):
@ -226,11 +187,11 @@ class TestClosestBranch(common.TransactionCase):
if b2_closest:
self.assertClosest(b2, closest[b2])
if noDuplicate:
self.assertNotEqual(build2.state, 'duplicate')
self.assertNotEqual(build2.local_state, 'duplicate')
self.assertFalse(build2.duplicate_id, "build on %s was detected as duplicate of build %s" % (self.branch_description(b2), build2.duplicate_id))
else:
self.assertEqual(build2.duplicate_id.id, build1.id, "build on %s wasn't detected as duplicate of build on %s" % (self.branch_description(b2), self.branch_description(b1)))
self.assertEqual(build2.state, 'duplicate')
self.assertEqual(build2.local_state, 'duplicate')
def assertNoDuplicate(self, branch1, branch2, b1_closest=None, b2_closest=None):
self.assertDuplicate(branch1, branch2, b1_closest=b1_closest, b2_closest=b2_closest, noDuplicate=True)

6
runbot/tests/test_cron.py
View File

@ -40,6 +40,9 @@ class Test_Cron(common.TransactionCase):
mock_fqdn.return_value = 'runbotx.foo.com'
mock_cron_period.return_value = 2
self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_update_frequency', 1)
self.Repo.create({'name': '/path/somewhere/disabled.git', 'mode': 'disabled'}) # create a disabled
self.Repo.search([]).write({'mode': 'disabled'}) # disable all depo, in case we have existing ones
local_repo = self.Repo.create({'name': '/path/somewhere/rep.git'}) # create active repo
ret = self.Repo._cron_fetch_and_schedule('runbotx.foo.com')
self.assertEqual(None, ret)
mock_update.assert_called_with(force=False)
@ -54,6 +57,9 @@ class Test_Cron(common.TransactionCase):
mock_fqdn.return_value = 'runbotx.foo.com'
mock_cron_period.return_value = 2
self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_update_frequency', 1)
self.Repo.create({'name': '/path/somewhere/disabled.git', 'mode': 'disabled'}) # create a disabled
self.Repo.search([]).write({'mode': 'disabled'}) # disable all depo, in case we have existing ones
local_repo = self.Repo.create({'name': '/path/somewhere/rep.git'}) # create active repo
ret = self.Repo._cron_fetch_and_build('runbotx.foo.com')
self.assertEqual(None, ret)
mock_scheduler.assert_called()

8
runbot/tests/test_frontend.py
View File

@ -42,8 +42,8 @@ class Test_Frontend(common.HttpCase):
'branch_id': branch.id,
'name': '%s0000ffffffffffffffffffffffffffff' % name,
'port': '1234',
'state': state,
'result': 'ok'
'local_state': state,
'local_result': 'ok'
})
def mocked_simple_repo_render(template, context):
@ -59,7 +59,7 @@ class Test_Frontend(common.HttpCase):
return Response()
mock_request.render = mocked_simple_repo_render
controller.repo()
controller.repo(repo=self.repo)
def mocked_repo_search_render(template, context):
dead_count = len([bu['name'] for b in context['branches'] for bu in b['builds'] if bu['name'].startswith('dead')])
@ -69,4 +69,4 @@ class Test_Frontend(common.HttpCase):
return Response()
mock_request.render = mocked_repo_search_render
controller.repo(search='dead')
controller.repo(repo=self.repo, search='dead')

55
runbot/tests/test_job_types.py
View File

@ -1,55 +0,0 @@
# -*- coding: utf-8 -*-
from time import localtime
from unittest.mock import patch
from odoo.tools.misc import DEFAULT_SERVER_DATETIME_FORMAT
from odoo.tests import common
class Test_Jobs(common.TransactionCase):
def setUp(self):
super(Test_Jobs, self).setUp()
self.Repo = self.env['runbot.repo']
self.repo = self.Repo.create({'name': 'bla@example.com:foo/bar', 'token': 'xxx'})
self.Branch = self.env['runbot.branch']
self.branch_master = self.Branch.create({
'repo_id': self.repo.id,
'name': 'refs/heads/master',
})
self.Build = self.env['runbot.build']
self.build = self.Build.create({
'branch_id': self.branch_master.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
'port' : '1234',
})
@patch('odoo.addons.runbot.models.build.docker_run')
@patch('odoo.addons.runbot.models.build.runbot_build._local_pg_createdb')
def test_job_10(self, mock_create_db, mock_docker_run):
""" Test that job10 is done or skipped depending on job_type """
# test that the default job_type value executes the tests
mock_docker_run.return_value = "Mocked run"
ret = self.Build._job_10_test_base(self.build, '/tmp/x.log')
self.assertEqual("Mocked run", ret, "A build with default job_type should run job_10")
# test skip when job_type is none
self.build.job_type = 'none'
ret = self.Build._job_10_test_base(self.build, '/tmp/x.log')
self.assertEqual(-2, ret, "A build with job_type 'none' should skip job_10")
# test skip when job_type is running
self.build.job_type = 'running'
ret = self.Build._job_10_test_base(self.build, '/tmp/x.log')
self.assertEqual(-2, ret, "A build with job_type 'running' should skip job_10")
# test run when job_type is testing
self.build.job_type = 'testing'
ret = self.Build._job_10_test_base(self.build, '/tmp/x.log')
self.assertEqual("Mocked run", ret, "A build with job_type 'testing' should run job_10")
# test run when job_type is all
self.build.job_type = 'all'
ret = self.Build._job_10_test_base(self.build, '/tmp/x.log')
self.assertEqual("Mocked run", ret, "A build with job_type 'all' should run job_10")

126
runbot/tests/test_jobs.py
View File

@ -1,126 +0,0 @@
# -*- coding: utf-8 -*-
import datetime
from time import localtime
from unittest.mock import patch
from odoo.tools.misc import DEFAULT_SERVER_DATETIME_FORMAT
from odoo.tests import common
class Test_Jobs(common.TransactionCase):
def setUp(self):
super(Test_Jobs, self).setUp()
self.Repo = self.env['runbot.repo']
self.repo = self.Repo.create({'name': 'bla@example.com:foo/bar', 'token': 'xxx'})
self.Branch = self.env['runbot.branch']
self.branch_master = self.Branch.create({
'repo_id': self.repo.id,
'name': 'refs/heads/master',
})
self.Build = self.env['runbot.build']
@patch('odoo.addons.runbot.models.repo.runbot_repo._domain')
@patch('odoo.addons.runbot.models.repo.runbot_repo._github')
@patch('odoo.addons.runbot.models.build.runbot_build._checkout')
def test_job_00_set_pending(self, mock_checkout, mock_github, mock_domain):
"""Test that job_00_init sets the pending status on github"""
mock_domain.return_value = 'runbotxx.somewhere.com'
build = self.Build.create({
'branch_id': self.branch_master.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
'host': 'runbotxx',
'port': '1234',
'state': 'testing',
'job': 'job_00_init',
'job_start': datetime.datetime.now(),
'job_end': False,
})
res = self.Build._job_00_init(build, '/tmp/x.log')
self.assertEqual(res, -2)
expected_status = {
'state': 'pending',
'target_url': 'http://runbotxx.somewhere.com/runbot/build/%s' % build.id,
'description': 'runbot build %s (runtime 0s)' % build.dest,
'context': 'ci/runbot'
}
mock_github.assert_called_with('/repos/:owner/:repo/statuses/d0d0caca0000ffffffffffffffffffffffffffff', expected_status, ignore_errors=True)
@patch('odoo.addons.runbot.models.build.docker_get_gateway_ip')
@patch('odoo.addons.runbot.models.repo.runbot_repo._domain')
@patch('odoo.addons.runbot.models.repo.runbot_repo._github')
@patch('odoo.addons.runbot.models.build.runbot_build._cmd')
@patch('odoo.addons.runbot.models.build.os.path.getmtime')
@patch('odoo.addons.runbot.models.build.time.localtime')
@patch('odoo.addons.runbot.models.build.docker_run')
@patch('odoo.addons.runbot.models.build.grep')
def test_job_29_failed(self, mock_grep, mock_docker_run, mock_localtime, mock_getmtime, mock_cmd, mock_github, mock_domain, mock_docker_get_gateway):
""" Test that a failed build sets the failure state on github """
a_time = datetime.datetime.now().strftime(DEFAULT_SERVER_DATETIME_FORMAT)
mock_grep.return_value = False
mock_docker_run.return_value = 2
now = localtime()
mock_localtime.return_value = now
mock_getmtime.return_value = None
mock_cmd.return_value = ([], [])
mock_domain.return_value = 'runbotxx.somewhere.com'
build = self.Build.create({
'branch_id': self.branch_master.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
'port' : '1234',
'state': 'testing',
'job_start': a_time,
'job_end': a_time
})
self.assertFalse(build.result)
self.Build._job_29_results(build, '/tmp/x.log')
self.assertEqual(build.result, 'ko')
expected_status = {
'state': 'failure',
'target_url': 'http://runbotxx.somewhere.com/runbot/build/%s' % build.id,
'description': 'runbot build %s (runtime 0s)' % build.dest,
'context': 'ci/runbot'
}
mock_github.assert_called_with('/repos/:owner/:repo/statuses/d0d0caca0000ffffffffffffffffffffffffffff', expected_status, ignore_errors=True)
@patch('odoo.addons.runbot.models.build.rfind')
@patch('odoo.addons.runbot.models.build.docker_get_gateway_ip')
@patch('odoo.addons.runbot.models.repo.runbot_repo._domain')
@patch('odoo.addons.runbot.models.repo.runbot_repo._github')
@patch('odoo.addons.runbot.models.build.runbot_build._cmd')
@patch('odoo.addons.runbot.models.build.os.path.getmtime')
@patch('odoo.addons.runbot.models.build.time.localtime')
@patch('odoo.addons.runbot.models.build.docker_run')
@patch('odoo.addons.runbot.models.build.grep')
def test_job_29_warned(self, mock_grep, mock_docker_run, mock_localtime, mock_getmtime, mock_cmd, mock_github, mock_domain, mock_docker_get_gateway, mock_rfind):
""" Test that a warn build sets the failure state on github """
def rfind_side_effect(logfile, regex):
return True if 'WARNING' in regex else False
a_time = datetime.datetime.now().strftime(DEFAULT_SERVER_DATETIME_FORMAT)
mock_rfind.side_effect = rfind_side_effect
mock_grep.return_value = True
mock_docker_run.return_value = 2
now = localtime()
mock_localtime.return_value = now
mock_getmtime.return_value = None
mock_cmd.return_value = ([], [])
mock_domain.return_value = 'runbotxx.somewhere.com'
build = self.Build.create({
'branch_id': self.branch_master.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
'port': '1234',
'state': 'testing',
'job_start': a_time,
'job_end': a_time
})
self.assertFalse(build.result)
self.Build._job_29_results(build, '/tmp/x.log')
self.assertEqual(build.result, 'warn')
expected_status = {
'state': 'failure',
'target_url': 'http://runbotxx.somewhere.com/runbot/build/%s' % build.id,
'description': 'runbot build %s (runtime 0s)' % build.dest,
'context': 'ci/runbot'
}
mock_github.assert_called_with('/repos/:owner/:repo/statuses/d0d0caca0000ffffffffffffffffffffffffffff', expected_status, ignore_errors=True)

25
runbot/tests/test_repo.py
View File

@ -70,8 +70,8 @@ class Test_Repo(common.TransactionCase):
build = self.env['runbot.build'].search([('repo_id', '=', repo.id), ('branch_id', '=', branch.id)])
self.assertEqual(build.subject, 'A nice subject')
self.assertEqual(build.state, 'pending')
self.assertFalse(build.result)
self.assertEqual(build.local_state, 'pending')
self.assertFalse(build.local_result)
# Simulate that a new commit is found in the other repo
self.commit_list = [('refs/heads/bidon',
@ -91,8 +91,8 @@ class Test_Repo(common.TransactionCase):
build = self.env['runbot.build'].search([('repo_id', '=', repo.id), ('branch_id', '=', branch.id)])
self.assertEqual(build.subject, 'A nice subject')
self.assertEqual(build.state, 'pending')
self.assertFalse(build.result)
self.assertEqual(build.local_state, 'pending')
self.assertFalse(build.local_result)
# A new commit is found in the first repo, the previous pending build should be skipped
self.commit_list = [('refs/heads/bidon',
@ -112,12 +112,12 @@ class Test_Repo(common.TransactionCase):
build = self.env['runbot.build'].search([('repo_id', '=', repo.id), ('branch_id', '=', branch.id), ('name', '=', 'b00b')])
self.assertEqual(build.subject, 'Another subject')
self.assertEqual(build.state, 'pending')
self.assertFalse(build.result)
self.assertEqual(build.local_state, 'pending')
self.assertFalse(build.local_result)
previous_build = self.env['runbot.build'].search([('repo_id', '=', repo.id), ('branch_id', '=', branch.id), ('name', '=', 'd0d0caca')])
self.assertEqual(previous_build.state, 'done', 'Previous pending build should be done')
self.assertEqual(previous_build.result, 'skipped', 'Previous pending build result should be skipped')
self.assertEqual(previous_build.local_state, 'done', 'Previous pending build should be done')
self.assertEqual(previous_build.local_result, 'skipped', 'Previous pending build result should be skipped')
@skip('This test is for performances. It needs a lot of real branches in DB to mean something')
@patch('odoo.addons.runbot.models.repo.runbot_repo._root')
@ -174,6 +174,7 @@ class Test_Repo_Scheduler(common.TransactionCase):
@patch('odoo.addons.runbot.models.repo.fqdn')
def test_repo_scheduler(self, mock_repo_fqdn, mock_schedule, mock_kill, mock_reap):
mock_repo_fqdn.return_value = 'test_host'
self.env['ir.config_parameter'].set_param('runbot.runbot_workers', 6)
Build_model = self.env['runbot.build']
builds = []
# create 6 builds that are testing on the host to verify that
@ -184,7 +185,7 @@ class Test_Repo_Scheduler(common.TransactionCase):
'name': build_name,
'port': '1234',
'build_type': 'normal',
'state': 'testing',
'local_state': 'testing',
'host': 'test_host'
})
builds.append(build)
@ -194,7 +195,7 @@ class Test_Repo_Scheduler(common.TransactionCase):
'name': 'sched_build',
'port': '1234',
'build_type': 'scheduled',
'state': 'pending',
'local_state': 'pending',
})
builds.append(scheduled_build)
# create the build that should be assigned once a slot is available
@ -203,7 +204,7 @@ class Test_Repo_Scheduler(common.TransactionCase):
'name': 'foobuild',
'port': '1234',
'build_type': 'normal',
'state': 'pending',
'local_state': 'pending',
})
builds.append(build)
self.foo_repo._scheduler()
@ -214,7 +215,7 @@ class Test_Repo_Scheduler(common.TransactionCase):
self.assertFalse(scheduled_build.host)
# give some room for the pending build
Build_model.search([('name', '=', 'a')]).write({'state': 'done'})
Build_model.search([('name', '=', 'a')]).write({'local_state': 'done'})
self.foo_repo._scheduler()
build.invalidate_cache()

17
runbot/tests/test_schedule.py
View File

@ -26,25 +26,26 @@ class TestSchedule(common.TransactionCase):
@patch('odoo.addons.runbot.models.build.os.makedirs')
@patch('odoo.addons.runbot.models.build.os.path.getmtime')
@patch('odoo.addons.runbot.models.build.docker_is_running')
def test_schedule_skip_running(self, mock_running, mock_getmtime, mock_makedirs, mock_localcleanup):
def test_schedule_mark_done(self, mock_running, mock_getmtime, mock_makedirs, mock_localcleanup):
""" Test that results are set even when job_30_run is skipped """
job_end_time = datetime.datetime.now()
mock_getmtime.return_value = job_end_time.timestamp()
build = self.Build.create({
'state': 'testing',
'local_state': 'testing',
'branch_id': self.branch.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
'port': '1234',
'host': 'runbotxx',
'job_start': datetime.datetime.now(),
'job_type': 'testing',
'job': 'job_20_test_all'
'config_id': self.env.ref('runbot.runbot_build_config_default').id,
'active_step': self.env.ref('runbot.runbot_build_config_step_run').id,
})
domain = [('repo_id', 'in', (self.repo.id, )), ('branch_id.job_type', '!=', 'none')]
domain = [('repo_id', 'in', (self.repo.id, ))]
domain_host = domain + [('host', '=', 'runbotxx')]
build_ids = self.Build.search(domain_host + [('state', 'in', ['testing', 'running', 'deathrow'])])
build_ids = self.Build.search(domain_host + [('local_state', 'in', ['testing', 'running', 'deathrow'])])
mock_running.return_value = False
self.assertEqual(build.local_state, 'testing')
build_ids._schedule()
self.assertEqual(build.state, 'done')
self.assertEqual(build.result, 'ko')
self.assertEqual(build.local_state, 'done')
self.assertEqual(build.local_result, 'ok')

10
runbot/views/assets.xml Normal file
View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<odoo>
<data>
<template id="assets_front_end" inherit_id="web.assets_frontend" name="runbot assets">
<xpath expr="." position="inside">
<link rel="stylesheet" href="/runbot/static/src/less/runbot.less"/>
</xpath>
</template>
</data>
</odoo>

10
runbot/views/branch_views.xml
View File

@ -16,14 +16,9 @@
<field name="pull_head_name"/>
<field name="sticky"/>
<field name="priority"/>
<field name="job_type"/>
<field name="coverage"/>
<field name="state"/>
<field name="modules"/>
<field name="job_timeout"/>
<!-- keep for next version
<field name="test_tags"/>
-->
<field name="config_id"/>
</group>
</sheet>
</form>
@ -40,8 +35,7 @@
<field name="name"/>
<field name="sticky"/>
<field name="priority"/>
<field name="job_type"/>
<field name="coverage"/>
<field name="config_id"/>
<field name="state"/>
</tree>
</field>

55
runbot/views/build_views.xml
View File

@ -4,14 +4,10 @@
<field name="model">runbot.build</field>
<field name="arch" type="xml">
<form string="Build">
<header>
<button name="reset" type="object" string="Reset"/>
</header>
<sheet>
<group>
<field name="repo_id"/>
<field name="branch_id"/>
<field name="sequence"/>
<field name="name"/>
<field name="date"/>
<field name="author"/>
@ -21,16 +17,25 @@
<field name="subject"/>
<field name="port"/>
<field name="dest"/>
<field name="state"/>
<field name="result"/>
<field name="local_state"/>
<field name="global_state"/>
<field name="local_result"/>
<field name="global_result"/>
<field name="triggered_result" groups="base.group_no_one"/>
<field name="pid"/>
<field name="host"/>
<field name="job_start"/>
<field name="job_end"/>
<field name="job_time"/>
<field name="job_age"/>
<field name="job_start" groups="base.group_no_one"/>
<field name="job_end" groups="base.group_no_one"/>
<field name="job_time" groups="base.group_no_one"/>
<field name="build_start"/>
<field name="build_end"/>
<field name="build_time"/>
<field name="build_age"/>
<field name="duplicate_id"/>
<field name="modules"/>
<field name="build_type" groups="base.group_no_one"/>
<field name="config_id" readonly="1"/>
<field name="config_id" groups="base.group_no_one"/>
</group>
</sheet>
</form>
@ -40,22 +45,20 @@
<field name="model">runbot.build</field>
<field name="arch" type="xml">
<tree string="Builds">
<field name="sequence"/>
<field name="repo_id"/>
<field name="dest"/>
<field name="date"/>
<field name="author"/>
<field name="committer"/>
<field name="state"/>
<field name="global_state"/>
<field name="global_result"/>
<field name="port"/>
<field name="job"/>
<field name="result"/>
<field name="coverage_result"/>
<field name="pid"/>
<field name="host"/>
<field name="job_start"/>
<field name="job_time"/>
<field name="job_age"/>
<field name="build_time"/>
<field name="build_age"/>
</tree>
</field>
</record>
@ -65,7 +68,7 @@
<field name="arch" type="xml">
<pivot string="Builds analysis">
<field name="create_date" interval="week" type="row"/>
<field name="state" type="col"/>
<field name="global_state" type="col"/>
</pivot>
</field>
</record>
@ -75,21 +78,21 @@
<search string="Search builds">
<field name="branch_id"/>
<field name="name"/>
<field name="state"/>
<field name="global_state"/>
<field name="dest"/>
<separator/>
<filter string="Pending" domain="[('state','=', 'pending')]"/>
<filter string="Testing" domain="[('state','=', 'testing')]"/>
<filter string="Running" domain="[('state','=', 'running')]"/>
<filter string="Done" domain="[('state','=','done')]"/>
<filter string="Duplicate" domain="[('state','=', 'duplicate')]"/>
<filter string="Deathrow" domain="[('state','=', 'deathrow')]"/>
<filter string="Pending" domain="[('global_state','=', 'pending')]"/>
<filter string="Testing" domain="[('global_state','in', ('testing', 'waiting'))]"/>
<filter string="Running" domain="[('global_state','=', 'running')]"/>
<filter string="Done" domain="[('global_state','=','done')]"/>
<filter string="Duplicate" domain="[('global_state','=', 'duplicate')]"/>
<filter string="Deathrow" domain="[('global_state','=', 'deathrow')]"/>
<separator />
<group expand="0" string="Group By...">
<filter string="Repo" domain="[]" context="{'group_by':'repo_id'}"/>
<filter string="Branch" domain="[]" context="{'group_by':'branch_id'}"/>
<filter string="Status" domain="[]" context="{'group_by':'state'}"/>
<filter string="Result" domain="[]" context="{'group_by':'result'}"/>
<filter string="Status" domain="[]" context="{'group_by':'global_state'}"/>
<filter string="Result" domain="[]" context="{'group_by':'global_result'}"/>
<filter string="Start" domain="[]" context="{'group_by':'job_start'}"/>
<filter string="Host" domain="[]" context="{'group_by':'host'}"/>
<filter string="Create Date" domain="[]" context="{'group_by':'create_date'}"/>

100
runbot/views/config_views.xml Normal file
View File

@ -0,0 +1,100 @@
<odoo>
<data>
<record id="config_form" model="ir.ui.view">
<field name="model">runbot.build.config</field>
<field name="arch" type="xml">
<form string="Build config">
<sheet>
<div attrs="{'invisible': [('protected', '=', False)]}">
<i class="fa fa-warning text-warning"/>This record is protected and can only be edited by config administrator.
</div>
<group>
<field name="name"/>
<field name="description"/>
<field name="step_order_ids">
<tree string="Step list" editable="bottom">
<field name="step_id"/>
<field name="sequence" groups="base.group_no_one"/>
<field name="sequence" widget="handle"/>
</tree>
</field>
<field name="update_github_state" readonly='1'/>
<field name="update_github_state" groups="base.group_no_one"/>
<field name="protected" groups="base.group_no_one"/>
</group>
</sheet>
</form>
</field>
</record>
<record id="config_step_form" model="ir.ui.view">
<field name="model">runbot.build.config.step</field>
<field name="arch" type="xml">
<form string="Build config step">
<sheet>
<div t-att-class="label label-warning" attrs="{'invisible': [('protected', '=', False)]}">
This record is protected and can only be edited by config administrator.
</div>
<group string="General settings">
<field name="name"/>
<field name="job_type"/>
<field name="protected" groups="base.group_no_one"/>
<field name="default_sequence" groups="base.group_no_one"/>
</group>
<group string="Python settings" attrs="{'invisible': [('job_type', 'not in', ('python'))]}">
<field name="python_code" widget="ace" options="{'mode': 'python'}"/>
<field name="running_job"/>
</group>
<group string="Test settings" attrs="{'invisible': [('job_type', 'not in', ('python', 'install_odoo'))]}">
<field name="create_db" groups="base.group_no_one"/>
<field name="install_modules"/>
<field name="db_name" groups="base.group_no_one"/>
<field name="cpu_limit" groups="base.group_no_one"/>
<field name="coverage"/>
<field name="test_enable" groups="base.group_no_one"/>
<field name="test_tags"/>
<field name="extra_params" groups="base.group_no_one"/>
</group>
<group string="Create settings" attrs="{'invisible': [('job_type', 'not in', ('python', 'create_build'))]}">
<field name="create_config_ids" widget="many2many_tags" options="{'no_create': True}" />
<field name="number_builds"/>
<field name="hide_build" groups="base.group_no_one"/>
<field name="force_build"/>
</group>
</sheet>
</form>
</field>
</record>
<record id="open_view_job_config_tree" model="ir.actions.act_window">
<field name="name">Build Configs</field>
<field name="res_model">runbot.build.config</field>
<field name="view_mode">tree,form</field>
</record>
<record id="open_view_job_tree" model="ir.actions.act_window">
<field name="name">Build Config Steps</field>
<field name="res_model">runbot.build.config.step</field>
<field name="view_mode">tree,form</field>
</record>
<menuitem
name="Build Configs"
id="runbot_menu_job_config_tree"
parent="runbot_menu_root"
sequence="30"
action="open_view_job_config_tree"
/>
<menuitem
name="Build Config Step"
id="runbot_menu_job_tree"
parent="runbot_menu_root"
sequence="31"
action="open_view_job_tree"
/>
</data>
</odoo>

1
runbot/views/repo_views.xml
View File

@ -22,6 +22,7 @@
<field name="token"/>
<field name="group_ids" widget="many2many_tags"/>
<field name="hook_time"/>
<field name="config_id"/>
</group>
</sheet>
</form>

2
runbot_cla/__init__.py
View File

@ -1 +1 @@
from . import runbot
from . import build_config

4
runbot_cla/__manifest__.py
View File

@ -6,5 +6,7 @@
'description': "Runbot CLA",
'author': 'Odoo SA',
'depends': ['runbot'],
'data': [],
'data': [
'data/runbot_build_config_data.xml',
],
}

19
runbot_cla/runbot.py → runbot_cla/build_config.py
View File

@ -5,17 +5,22 @@ import io
import logging
import re
from odoo.addons.runbot.models.build import runbot_job
from odoo import models
from odoo import models, fields
_logger = logging.getLogger(__name__)
class runbot_build(models.Model):
_inherit = "runbot.build"
class Job(models.Model):
_inherit = "runbot.build.config.step"
@runbot_job('testing')
def _job_05_check_cla(self, build, log_path):
job_type = fields.Selection(selection_add=[('cla_check', 'Check cla')])
def _run_step(self, build, log_path):
if self.job_type != 'cla_check':
return self._runbot_cla_check(build, log_path)
return super(Job, self)._run_step(build, log_path)
def _runbot_cla_check(self, build, log_path):
cla_glob = glob.glob(build._path("doc/cla/*/*.md"))
if cla_glob:
description = "%s Odoo CLA signature check" % build.author
@ -27,7 +32,7 @@ class runbot_build(models.Model):
state = "success"
else:
try:
cla = ''.join(io.open(f,encoding='utf-8').read() for f in cla_glob)
cla = ''.join(io.open(f, encoding='utf-8').read() for f in cla_glob)
if cla.lower().find(email) != -1:
state = "success"
except UnicodeDecodeError:

12
runbot_cla/data/runbot_build_config_data.xml Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="utf-8"?>
<odoo>
<record id="runbot_build_config_step_check_cla" model="runbot.build.config.step">
<field name="name">cla_check</field>
<field name="job_type">cla_check</field>
<field name="protected" eval="True"/>
</record>
<record id="runbot.runbot_build_config_default" model="runbot.build.config">
<field name="step_order_ids" eval="[(0, 0, {'step_id': ref('runbot_build_config_step_check_cla')})]"/>
</record>
</odoo>