From 8c73e6a90133c8915ced619a31e48b10fb01b70d Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Tue, 18 Dec 2018 13:54:24 +0100 Subject: [PATCH] [IMP] runbot: add a job_type on branch and build Since the runbot_merge module, some branches does not need to be built. For example the tmp.* branches. Some other branches does need to be tested but it could be useless to keep them running. For example the staging branches. Finally, some builds are generated by server actions during the night. Those builds does not need to be kept running despite the branch configuration. For example, the master branch can be configured to create builds with testing and running but nightly multiple builds can be generated with testing only. For that purpose, this commit adds a job_type selection field on the branch. That way, a branch can be configured by selecting the type of jobs wanted. A same kind of job_type was also added on the build that uses the branch's value if nothing is specified at build creation. A decorator is used on the job_ methods to specify their job types. For example, a job method decorated by 'testing' will run if the branch/build job_type is 'testing' or 'all'. --- runbot/__manifest__.py | 2 +- runbot/models/branch.py | 6 ++++ runbot/models/build.py | 40 ++++++++++++++++++++++--- runbot/models/repo.py | 2 +- runbot/tests/__init__.py | 3 +- runbot/tests/test_branch.py | 1 + runbot/tests/test_build.py | 27 +++++++++++++++++ runbot/tests/test_job_types.py | 55 ++++++++++++++++++++++++++++++++++ runbot/views/branch_views.xml | 2 ++ runbot_cla/runbot.py | 2 ++ 10 files changed, 133 insertions(+), 7 deletions(-) create mode 100644 runbot/tests/test_job_types.py diff --git a/runbot/__manifest__.py b/runbot/__manifest__.py index 8650b1d6..b2f64666 100644 --- a/runbot/__manifest__.py +++ b/runbot/__manifest__.py @@ -6,7 +6,7 @@ 'author': "Odoo SA", 'website': "http://runbot.odoo.com", 'category': 'Website', - 'version': '2.5', + 'version': '2.6', 'depends': ['website', 'base'], 'data': [ 'security/runbot_security.xml', diff --git a/runbot/models/branch.py b/runbot/models/branch.py index bd5b99ac..51b16631 100644 --- a/runbot/models/branch.py +++ b/runbot/models/branch.py @@ -27,6 +27,12 @@ class runbot_branch(models.Model): modules = fields.Char("Modules to Install", help="Comma-separated list of modules to install and test.") job_timeout = fields.Integer('Job Timeout (minutes)', help='For default timeout: Mark it zero') priority = fields.Boolean('Build priority', default=False) + job_type = fields.Selection([ + ('testing', 'Testing jobs only'), + ('running', 'Running job only'), + ('all', 'All jobs'), + ('none', 'Do not execute jobs') + ], required=True, default='all') @api.depends('name') def _get_branch_infos(self): diff --git a/runbot/models/build.py b/runbot/models/build.py index b05e9256..748f78e9 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -25,6 +25,19 @@ re_job = re.compile('_job_\d') _logger = logging.getLogger(__name__) +def runbot_job(*accepted_job_types): + """ Decorator for runbot_build _job_x methods to filter build jobs """ + accepted_job_types += ('all', ) + + def job_decorator(func): + def wrapper(self, build, log_path): + if build.job_type == 'none' or build.job_type not in accepted_job_types: + build._log(func.__name__, 'Skipping job') + return -2 + return func(self, build, log_path) + return wrapper + return job_decorator + class runbot_build(models.Model): _name = "runbot.build" @@ -74,6 +87,13 @@ class runbot_build(models.Model): ], default='normal', string='Build type') + job_type = fields.Selection([ + ('testing', 'Testing jobs only'), + ('running', 'Running job only'), + ('all', 'All jobs'), + ('none', 'Do not execute jobs'), + ], + ) def copy(self, values=None): raise UserError("Cannot duplicate build!") @@ -81,6 +101,8 @@ class runbot_build(models.Model): def create(self, vals): build_id = super(runbot_build, self).create(vals) extra_info = {'sequence': build_id.id} + job_type = vals['job_type'] if 'job_type' in vals else build_id.branch_id.job_type + extra_info.update({'job_type': job_type}) context = self.env.context # detect duplicate @@ -733,6 +755,7 @@ class runbot_build(models.Model): # Jobs definitions # They all need "build log_path" parameters + @runbot_job('testing', 'running') def _job_00_init(self, build, log_path): build._log('init', 'Init build environment') # notify pending build - avoid confusing users by saying nothing @@ -740,12 +763,14 @@ class runbot_build(models.Model): build._checkout() return -2 + @runbot_job('testing', 'running') def _job_02_docker_build(self, build, log_path): """Build the docker image""" build._log('docker_build', 'Building docker image') docker_build(log_path, build._path()) return -2 + @runbot_job('testing') def _job_10_test_base(self, build, log_path): build._log('test_base', 'Start test base module') # run base test @@ -758,14 +783,18 @@ class runbot_build(models.Model): cmd.extend(shlex.split(build.extra_params)) return docker_run(cmd, log_path, build._path(), build._get_docker_name(), cpu_limit=600) + @runbot_job('testing', 'running') def _job_20_test_all(self, build, log_path): - build._log('test_all', 'Start test all modules') + cpu_limit = 2400 self._local_pg_createdb("%s-all" % build.dest) cmd, mods = build._cmd() - if grep(build._server("tools/config.py"), "test-enable"): - cmd.append("--test-enable") - cmd += ['-d', '%s-all' % build.dest, '-i', mods, '--stop-after-init', '--log-level=test', '--max-cron-threads=0'] + build._log('test_all', 'Start test all modules') + if grep(build._server("tools/config.py"), "test-enable") and build.job_type in ('testing', 'all'): + cmd.extend(['--test-enable', '--log-level=test']) + else: + build._log('test_all', 'Installing modules without testing') + cmd += ['-d', '%s-all' % build.dest, '-i', mods, '--stop-after-init', '--max-cron-threads=0'] if build.extra_params: cmd.extend(build.extra_params.split(' ')) if build.coverage: @@ -782,6 +811,7 @@ class runbot_build(models.Model): build.write({'job_start': now()}) return docker_run(cmd, log_path, build._path(), build._get_docker_name(), cpu_limit=cpu_limit) + @runbot_job('testing') def _job_21_coverage_html(self, build, log_path): if not build.coverage: return -2 @@ -791,6 +821,7 @@ class runbot_build(models.Model): cmd = [ get_py_version(build), "-m", "coverage", "html", "-d", "/data/build/coverage", "--ignore-errors"] return docker_run(cmd, log_path, build._path(), build._get_docker_name()) + @runbot_job('testing') def _job_22_coverage_result(self, build, log_path): if not build.coverage: return -2 @@ -805,6 +836,7 @@ class runbot_build(models.Model): build._log('coverage_result', 'Coverage file not found') return -2 # nothing to wait for + @runbot_job('running') def _job_30_run(self, build, log_path): # adjust job_end to record an accurate job_20 job_time build._log('run', 'Start running build %s' % build.dest) diff --git a/runbot/models/repo.py b/runbot/models/repo.py index c932fea5..c702757c 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -248,7 +248,7 @@ class runbot_repo(models.Model): host = fqdn() Build = self.env['runbot.build'] - domain = [('repo_id', 'in', ids)] + domain = [('repo_id', 'in', ids), ('branch_id.job_type', '!=', 'none')] domain_host = domain + [('host', '=', host)] # schedule jobs (transitions testing -> running, kill jobs, ...) diff --git a/runbot/tests/__init__.py b/runbot/tests/__init__.py index 809caab7..79c8d97b 100644 --- a/runbot/tests/__init__.py +++ b/runbot/tests/__init__.py @@ -2,4 +2,5 @@ from . import test_repo from . import test_branch from . import test_build from . import test_jobs -from . import test_frontend \ No newline at end of file +from . import test_frontend +from . import test_job_types diff --git a/runbot/tests/test_branch.py b/runbot/tests/test_branch.py index c626a1b8..fb9baee2 100644 --- a/runbot/tests/test_branch.py +++ b/runbot/tests/test_branch.py @@ -22,6 +22,7 @@ class Test_Branch(common.TransactionCase): self.assertEqual(branch.branch_name, 'master') self.assertEqual(branch.branch_url, 'https://example.com/foo/bar/tree/master') + self.assertEqual(branch.job_type, 'all') @patch('odoo.addons.runbot.models.repo.runbot_repo._github') def test_pull_request(self, mock_github): diff --git a/runbot/tests/test_build.py b/runbot/tests/test_build.py index b13f9bea..4897f417 100644 --- a/runbot/tests/test_build.py +++ b/runbot/tests/test_build.py @@ -112,6 +112,33 @@ class Test_Build(common.TransactionCase): self.assertEqual(dev_build.state, 'duplicate') self.assertEqual(dev_build.duplicate_id.id, pr_build.id) + def test_build_job_type_from_branch_default(self): + """test build job_type is computed from branch default job_type""" + build = self.Build.create({ + 'branch_id': self.branch.id, + 'name': 'd0d0caca0000ffffffffffffffffffffffffffff', + }) + self.assertEqual(build.job_type, 'all', "job_type should be the same as the branch") + + def test_build_job_type_from_branch_none(self): + """test build job_type is computed from branch""" + self.branch.job_type = 'none' + build = self.Build.create({ + 'branch_id': self.branch.id, + 'name': 'd0d0caca0000ffffffffffffffffffffffffffff', + }) + self.assertEqual(build.job_type, 'none', "job_type should be the same as the branch") + + def test_build_job_type_can_be_set(self): + """test build job_type can be set to something different than the one on the branch""" + self.branch.job_type = 'running' + build = self.Build.create({ + 'branch_id': self.branch.id, + 'name': 'd0d0caca0000ffffffffffffffffffffffffffff', + 'job_type': 'testing' + }) + self.assertEqual(build.job_type, 'testing', "job_type should be the one set on the build") + @patch('odoo.addons.runbot.models.branch.runbot_branch._is_on_remote') def test_closest_branch_01(self, mock_is_on_remote): """ test find a matching branch in a target repo based on branch name """ diff --git a/runbot/tests/test_job_types.py b/runbot/tests/test_job_types.py new file mode 100644 index 00000000..f25dced9 --- /dev/null +++ b/runbot/tests/test_job_types.py @@ -0,0 +1,55 @@ + +# -*- coding: utf-8 -*- +from time import localtime +from unittest.mock import patch +from odoo.tools.misc import DEFAULT_SERVER_DATETIME_FORMAT +from odoo.tests import common + + +class Test_Jobs(common.TransactionCase): + + + def setUp(self): + super(Test_Jobs, self).setUp() + self.Repo = self.env['runbot.repo'] + self.repo = self.Repo.create({'name': 'bla@example.com:foo/bar', 'token': 'xxx'}) + self.Branch = self.env['runbot.branch'] + self.branch_master = self.Branch.create({ + 'repo_id': self.repo.id, + 'name': 'refs/heads/master', + }) + self.Build = self.env['runbot.build'] + self.build = self.Build.create({ + 'branch_id': self.branch_master.id, + 'name': 'd0d0caca0000ffffffffffffffffffffffffffff', + 'port' : '1234', + }) + + @patch('odoo.addons.runbot.models.build.docker_run') + @patch('odoo.addons.runbot.models.build.runbot_build._local_pg_createdb') + def test_job_10(self, mock_create_db, mock_docker_run): + """ Test that job10 is done or skipped depending on job_type """ + # test that the default job_type value executes the tests + mock_docker_run.return_value = "Mocked run" + ret = self.Build._job_10_test_base(self.build, '/tmp/x.log') + self.assertEqual("Mocked run", ret, "A branch with default job_type should run job_10") + + # test skip when job_type is none + self.build.job_type = 'none' + ret = self.Build._job_10_test_base(self.build, '/tmp/x.log') + self.assertEqual(-2, ret, "A branch with job_type 'none' should skip job_10") + + # test skip when job_type is running + self.build.job_type = 'running' + ret = self.Build._job_10_test_base(self.build, '/tmp/x.log') + self.assertEqual(-2, ret, "A branch with job_type 'running' should skip job_10") + + # test run when job_type is testing + self.build.job_type = 'testing' + ret = self.Build._job_10_test_base(self.build, '/tmp/x.log') + self.assertEqual("Mocked run", ret, "A branch with job_type 'testing' should run job_10") + + # test run when job_type is all + self.build.job_type = 'all' + ret = self.Build._job_10_test_base(self.build, '/tmp/x.log') + self.assertEqual("Mocked run", ret, "A branch with job_type 'all' should run job_10") \ No newline at end of file diff --git a/runbot/views/branch_views.xml b/runbot/views/branch_views.xml index 1c20f338..870fbe86 100644 --- a/runbot/views/branch_views.xml +++ b/runbot/views/branch_views.xml @@ -16,6 +16,7 @@ + @@ -39,6 +40,7 @@ + diff --git a/runbot_cla/runbot.py b/runbot_cla/runbot.py index aa937f0f..f79202cb 100644 --- a/runbot_cla/runbot.py +++ b/runbot_cla/runbot.py @@ -5,6 +5,7 @@ import io import logging import re +from odoo.addons.runbot.models.build import runbot_job from odoo import models _logger = logging.getLogger(__name__) @@ -13,6 +14,7 @@ _logger = logging.getLogger(__name__) class runbot_build(models.Model): _inherit = "runbot.build" + @runbot_job('testing') def _job_05_check_cla(self, build, log_path): cla_glob = glob.glob(build._path("doc/cla/*/*.md")) if cla_glob: