From 24d35988a45c61bb4ba1e6080a815ba7269a2fd8 Mon Sep 17 00:00:00 2001
From: xdo <35262360+Xavier-Do@users.noreply.github.com>
Date: Tue, 27 Jun 2023 14:13:12 +0200
Subject: [PATCH] [IMP] runbot: add auto restore for custom trigger (#784)
One of the most common custom trigger is to restore a build before
starting some test, either to create a multibuild or make the execution
and debug of some test faster.
It is somethimes tedious to use because we need to give an url of a
build to restore. This build must correspond to the right commits,
must still exixt, ... this means that the dump url must be adapted
everytime a branch is rebased.
The way the dump_url is defined is by going on the last batch, following
the link to the `base_reference_batch_id`, finding a slot corresponding
to the right repo set, (ex: Custom enterprise -> enterprise), and
copying the dump_url in this build.
The base_reference_batch_id is eay to automated but we have to find the
right trigger, this is now a parameter of the custom trigger wizard.
There are actually 2 strategy now to define how to download the dump:
- `url`, using `restore_ dump_url`
- `auto`, using `restore_trigger_id` and `restore_database_suffix`
To ease the setup, a `restore_trigger_id` is added on a trigger, so that
when selecting a trigger, lets say `Custom enterprise`, the defined
`trigger.restore_trigger_id` is automatically chosen for the
`custom_trigger.restore_trigger_id` and the `restore_mode` is setted to
auto.
Two actions are also added to the header of a bundle, a shorcut to
setup a multi build (restore in children) or a restore and test build
(restore in parent).
---
runbot/models/build_config.py | 27 +++-
runbot/models/bundle.py | 32 +++++
runbot/models/custom_trigger.py | 110 +++++++++++++---
runbot/models/repo.py | 12 +-
runbot/tests/common.py | 2 +-
runbot/tests/test_build_config_step.py | 60 +++++++++
runbot/views/bundle_views.xml | 4 +
runbot/views/custom_trigger_wizard_views.xml | 32 +++--
runbot/views/repo_views.xml | 132 ++++++++++++-------
9 files changed, 329 insertions(+), 82 deletions(-)
diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py
index 5f70e5f5..3f8564be 100644
--- a/runbot/models/build_config.py
+++ b/runbot/models/build_config.py
@@ -741,17 +741,36 @@ class ConfigStep(models.Model):
if 'dump_url' in params.config_data:
dump_url = params.config_data['dump_url']
zip_name = dump_url.split('/')[-1]
- build._log('test-migration', 'Restoring db [%s](%s)' % (zip_name, dump_url), log_type='markdown')
+ build._log('_run_restore', f'Restoring db [{zip_name}]({dump_url})', log_type='markdown')
suffix = 'all'
else:
- download_db_suffix = params.dump_db.db_suffix or self.restore_download_db_suffix
- dump_build = params.dump_db.build_id or build.parent_id
+ if 'dump_trigger_id' in params.config_data:
+ dump_trigger = self.env['runbot.trigger'].browse(params.config_data['dump_trigger_id'])
+ dump_suffix = params.config_data.get('dump_suffix', 'all')
+ base_batch = build.params_id.create_batch_id.base_reference_batch_id
+ reference_build = base_batch.slot_ids.filtered(lambda s: s.trigger_id == dump_trigger).mapped('build_id')
+ if not reference_build:
+ build._log('_run_restore', f'No reference build found in batch {base_batch.id} for trigger {dump_trigger.name}', log_type='markdown', level='ERROR')
+ build._kill(result='ko')
+ return
+ if reference_build.local_state not in ('done', 'running'):
+ build._log('_run_restore', f'Reference build [{reference_build.id}]({reference_build.build_url} is not yet finished, database may not exist', log_type='markdown', level='WARNING')
+ dump_db = reference_build.database_ids.filtered(lambda d: d.db_suffix == dump_suffix)
+ if not dump_db:
+ build._log('_run_restore', f'No dump with suffix {dump_suffix} found in build [{reference_build.id}]({reference_build.build_url})', log_type='markdown', level='ERROR')
+ build._kill(result='ko')
+ return
+ else:
+ dump_db = params.dump_db
+
+ download_db_suffix = dump_db.db_suffix or self.restore_download_db_suffix
+ dump_build = dump_db.build_id or build.parent_id
assert download_db_suffix and dump_build
download_db_name = '%s-%s' % (dump_build.dest, download_db_suffix)
zip_name = '%s.zip' % download_db_name
dump_url = '%s%s' % (dump_build.http_log_url(), zip_name)
build._log('test-migration', 'Restoring dump [%s](%s) from build [%s](%s)' % (zip_name, dump_url, dump_build.id, dump_build.build_url), log_type='markdown')
- restore_suffix = self.restore_rename_db_suffix or params.dump_db.db_suffix or suffix
+ restore_suffix = self.restore_rename_db_suffix or dump_db.db_suffix or suffix
assert restore_suffix
restore_db_name = '%s-%s' % (build.dest, restore_suffix)
diff --git a/runbot/models/bundle.py b/runbot/models/bundle.py
index 3355e5b8..7710eef7 100644
--- a/runbot/models/bundle.py
+++ b/runbot/models/bundle.py
@@ -248,3 +248,35 @@ class Bundle(models.Model):
for branch in self.branch_ids.sorted(key=lambda b: (b.is_pr)):
branch_groups[branch.remote_id.repo_id].append(branch)
return branch_groups
+
+ def generate_custom_trigger_multi_action(self):
+ context = {
+ 'default_bundle_id': self.id,
+ 'default_config_id': self.env.ref('runbot.runbot_build_config_custom_multi').id,
+ 'default_child_config_id': self.env.ref('runbot.runbot_build_config_restore_and_test').id,
+ 'default_extra_params': False,
+ 'default_child_extra_params': '--test-tags /module.test_method',
+ 'default_number_build': 10,
+ }
+ return self._generate_custom_trigger_action(context)
+
+ def generate_custom_trigger_restore_action(self):
+ context = {
+ 'default_bundle_id': self.id,
+ 'default_config_id': self.env.ref('runbot.runbot_build_config_restore_and_test').id,
+ 'default_child_config_id': False,
+ 'default_extra_params': '--test-tags /module.test_method',
+ 'default_child_extra_params': False,
+ 'default_number_build': 0,
+ }
+ return self._generate_custom_trigger_action(context)
+
+ def _generate_custom_trigger_action(self, context):
+ return {
+ 'type': 'ir.actions.act_window',
+ 'name': 'Generate custom trigger',
+ 'view_mode': 'form',
+ 'res_model': 'runbot.trigger.custom.wizard',
+ 'target': 'new',
+ 'context': context,
+ }
diff --git a/runbot/models/custom_trigger.py b/runbot/models/custom_trigger.py
index cdb4955a..0e79be60 100644
--- a/runbot/models/custom_trigger.py
+++ b/runbot/models/custom_trigger.py
@@ -26,57 +26,135 @@ class CustomTriggerWizard(models.TransientModel):
_name = 'runbot.trigger.custom.wizard'
_description = 'Custom trigger Wizard'
+ # minimal config options
bundle_id = fields.Many2one('runbot.bundle', "Bundle")
project_id = fields.Many2one(related='bundle_id.project_id', string='Project')
trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', project_id)]")
config_id = fields.Many2one('runbot.build.config', string="Config id", default=lambda self: self.env.ref('runbot.runbot_build_config_custom_multi'))
+ # base options
config_data = JsonDictField("Config data")
+ extra_params = fields.Char('Extra params', default='')
+ # restore options
+ restore_mode = fields.Selection([('auto', 'Auto'), ('url', 'Dump url')])
+ restore_dump_url = fields.Char('Dump url for children')
+ restore_trigger_id = fields.Many2one('runbot.trigger', 'Trigger to restore a dump', domain="[('project_id', '=', project_id), ('manual', '=', False)]")
+ restore_database_suffix = fields.Char('Database suffix to restore', default='all')
+
+ # create multi options
number_build = fields.Integer('Number builds for config multi', default=10)
-
child_extra_params = fields.Char('Extra params for children', default='--test-tags /module.test_method')
- child_dump_url = fields.Char('Dump url for children')
child_config_id = fields.Many2one('runbot.build.config', 'Config for children', default=lambda self: self.env.ref('runbot.runbot_build_config_restore_and_test'))
warnings = fields.Text('Warnings', readonly=True)
- @api.onchange('child_extra_params', 'child_dump_url', 'child_config_id', 'number_build', 'config_id', 'trigger_id')
+ has_create_step = fields.Boolean("Hase create step", compute="_compute_has_create_step")
+ has_restore_step = fields.Boolean("Hase restore step", compute="_compute_has_restore_step")
+ has_child_with_restore_step = fields.Boolean("Child config has create step", compute="_compute_has_child_with_restore_step")
+
+ @api.depends('config_id')
+ def _compute_has_create_step(self):
+ for record in self:
+ record.has_create_step = any(step.job_type == 'create_build' for step in self.config_id.step_ids())
+
+ @api.depends('config_id')
+ def _compute_has_restore_step(self):
+ for record in self:
+ record.has_restore_step = any(step.job_type == 'restore' for step in self.config_id.step_ids())
+
+ @api.depends('child_config_id')
+ def _compute_has_child_with_restore_step(self):
+ for record in self:
+ record.has_child_with_restore_step = record.child_config_id and any(step.job_type == 'restore' for step in self.child_config_id.step_ids())
+
+ @api.onchange('child_extra_params', 'restore_dump_url', 'config_id', 'child_config_id', 'number_build', 'config_id', 'restore_mode', 'restore_database_suffix', 'restore_trigger_id')
def _onchange_warnings(self):
for wizard in self:
_warnings = []
+
+ if not wizard.trigger_id:
+ _warnings.append(f'No trigger id given (required and may automatically fix other issues)')
+
if wizard._get_existing_trigger():
_warnings.append(f'A custom trigger already exists for trigger {wizard.trigger_id.name} and will be unlinked')
- if wizard.child_dump_url or wizard.child_extra_params or wizard.child_config_id or wizard.number_build:
- if not any(step.job_type == 'create_build' for step in wizard.config_id.step_ids()):
- _warnings.append('Some multi builds params are given but config as no create step')
+ if wizard.restore_mode:
+ if (not wizard.has_restore_step and not wizard.has_child_with_restore_step):
+ _warnings.append('A restore mode is defined but no config has a restore step')
+ elif not wizard.restore_mode:
+ if wizard.has_restore_step :
+ _warnings.append('Config has a restore step but no restore mode is given')
+ if wizard.has_child_with_restore_step:
+ _warnings.append('Child config has a restore step but no restore mode is given')
+ elif wizard.restore_mode == "url":
+ if not wizard.restore_dump_url:
+ _warnings.append('The restore mode is url but no dump_url is given')
+ elif wizard.restore_mode == "auto":
+ if not wizard.restore_trigger_id:
+ _warnings.append('The restore mode is auto but no restore trigger is given')
+ if not wizard.restore_database_suffix:
+ _warnings.append('The restore mode is auto but no db suffix is given')
- if wizard.child_dump_url and not any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()):
- _warnings.append('A dump_url is defined but child config has no restore step')
-
- if not wizard.child_dump_url and any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()):
- _warnings.append('Child config has a restore step but no dump_url is given')
+ if wizard.has_create_step:
+ if not wizard.child_config_id:
+ _warnings.append('Config has a create step nut no child config given')
+ if not wizard.child_extra_params:
+ _warnings.append('Config has a create step nut no child extra param given')
+ if wizard.extra_params:
+ _warnings.append('You may change `Extra params` to `Extra params for children`')
+ else:
+ if wizard.child_extra_params:
+ _warnings.append('Extra params for children given but config has no create step')
+ if wizard.child_config_id:
+ _warnings.append('Config for children given but config has no create step')
+ if not wizard.extra_params:
+ _warnings.append('No extra params are given')
if not wizard.trigger_id.manual:
_warnings.append("This custom trigger will replace an existing non manual trigger. The ci won't be sent anymore")
wizard.warnings = '\n'.join(_warnings)
- @api.onchange('number_build', 'child_extra_params', 'child_dump_url', 'child_config_id')
- def _onchange_config_data(self):
+ @api.onchange('trigger_id')
+ def _onchange_trigger_id(self):
for wizard in self:
- wizard.config_data = self._get_config_data()
+ if wizard.trigger_id:
+ wizard.restore_trigger_id = wizard.trigger_id.restore_trigger_id
+ if wizard.restore_trigger_id and not wizard.restore_mode:
+ wizard.restore_mode = 'auto'
+ self._onchange_config_data()
+ self._onchange_warnings()
+
+ @api.onchange('number_build', 'child_extra_params', 'restore_dump_url', 'child_config_id', 'restore_trigger_id', 'restore_database_suffix', 'restore_mode')
+ def _onchange_config_data(self):
+ for wizard in self:
+ wizard.config_data = self._get_config_data()
def _get_config_data(self):
config_data = {}
if self.number_build:
config_data['number_build'] = self.number_build
+ if self.extra_params:
+ config_data['extra_params'] = self.extra_params
child_data = {}
if self.child_extra_params:
child_data['extra_params'] = self.child_extra_params
- if self.child_dump_url:
- child_data['config_data'] = {'dump_url': self.child_dump_url}
+ if self.restore_mode:
+ restore_params = {}
+ if self.restore_mode == 'url':
+ if self.restore_dump_url:
+ restore_params['dump_url'] = self.restore_dump_url
+ else:
+ if self.restore_trigger_id:
+ restore_params['dump_trigger_id'] = self.restore_trigger_id.id
+ if self.restore_database_suffix:
+ restore_params['dump_suffix'] = self.restore_database_suffix
+ if self.has_child_with_restore_step:
+ child_data['config_data'] = restore_params
+ if not self.has_child_with_restore_step or self.has_restore_step:
+ config_data.update(restore_params)
+
if self.child_config_id:
child_data['config_id'] = self.child_config_id.id
if child_data:
diff --git a/runbot/models/repo.py b/runbot/models/repo.py
index 973d74f0..390fea59 100644
--- a/runbot/models/repo.py
+++ b/runbot/models/repo.py
@@ -38,22 +38,23 @@ class Trigger(models.Model):
sequence = fields.Integer('Sequence')
name = fields.Char("Name")
description = fields.Char("Description", help="Informative description")
- project_id = fields.Many2one('runbot.project', string="Project id", required=True, default=lambda self: self.env.ref('runbot.main_project', raise_if_not_found=False))
+ project_id = fields.Many2one('runbot.project', string="Project id", required=True)
repo_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_triggers', string="Triggers", domain="[('project_id', '=', project_id)]")
dependency_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_dependencies', string="Dependencies")
config_id = fields.Many2one('runbot.build.config', string="Config", required=True)
batch_dependent = fields.Boolean('Batch Dependent', help="Force adding batch in build parameters to make it unique and give access to bundle")
- ci_context = fields.Char("Ci context", default='ci/runbot', tracking=True)
+ ci_context = fields.Char("CI context", tracking=True)
category_id = fields.Many2one('runbot.category', default=lambda self: self.env.ref('runbot.default_category', raise_if_not_found=False))
version_domain = fields.Char(string="Version domain")
hide = fields.Boolean('Hide trigger on main page')
manual = fields.Boolean('Only start trigger manually', default=False)
+ restore_trigger_id = fields.Many2one('runbot.trigger', string='Restore Trigger ID for custom triggers', help="Mainly usefull to automatically define where to find a reference database when creating a custom trigger", tracking=True)
upgrade_dumps_trigger_id = fields.Many2one('runbot.trigger', string='Template/complement trigger', tracking=True)
upgrade_step_id = fields.Many2one('runbot.build.config.step', compute="_compute_upgrade_step_id", store=True)
- ci_url = fields.Char("ci url")
- ci_description = fields.Char("ci description")
+ ci_url = fields.Char("CI url")
+ ci_description = fields.Char("CI description")
has_stats = fields.Boolean('Has a make_stats config step', compute="_compute_has_stats", store=True)
team_ids = fields.Many2many('runbot.team', string="Runbot Teams", help="Teams responsible of this trigger, mainly usefull for nightly")
@@ -276,8 +277,7 @@ class Repo(models.Model):
main_remote_id = fields.Many2one('runbot.remote', "Main remote", tracking=True)
remote_ids = fields.One2many('runbot.remote', 'repo_id', "Remotes")
project_id = fields.Many2one('runbot.project', required=True, tracking=True,
- help="Default bundle project to use when pushing on this repos",
- default=lambda self: self.env.ref('runbot.main_project', raise_if_not_found=False))
+ help="Default bundle project to use when pushing on this repos")
# -> not verry usefull, remove it? (iterate on projects or contraints triggers:
# all trigger where a repo is used must be in the same project.
modules = fields.Char("Modules to install", help="Comma-separated list of modules to install and test.", tracking=True)
diff --git a/runbot/tests/common.py b/runbot/tests/common.py
index fd89543d..40d67137 100644
--- a/runbot/tests/common.py
+++ b/runbot/tests/common.py
@@ -114,7 +114,7 @@ class RunbotCase(TransactionCase):
'is_pr': False,
'head': self.initial_server_commit.id,
})
- self.branch_server.bundle_id # compute
+ self.master_bundle = self.branch_server.bundle_id # compute
self.dev_bundle = self.Bundle.create({
'name': 'master-dev-tri',
'project_id': self.project.id
diff --git a/runbot/tests/test_build_config_step.py b/runbot/tests/test_build_config_step.py
index 09fe6da2..18bbba50 100644
--- a/runbot/tests/test_build_config_step.py
+++ b/runbot/tests/test_build_config_step.py
@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
from unittest.mock import patch, mock_open
from odoo import Command
+from odoo.tools import mute_logger
from odoo.exceptions import UserError
from odoo.addons.runbot.common import RunbotException
from .common import RunbotCase
@@ -198,6 +199,65 @@ class TestCodeowner(TestBuildConfigStepCommon):
'Requesting review for pull request [base/server:1234](https://example.com/base/server/pull/1234): codeowner-team, team_01, team_02, team_js, team_py'
])
+class TestBuildConfigStepRestore(TestBuildConfigStepCommon):
+
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ cls.restore_config_step = cls.env['runbot.build.config.step'].create({
+ 'name': 'restore',
+ 'job_type': 'restore',
+ })
+ cls.restore_config = cls.env['runbot.build.config'].create({
+ 'name': 'Restore',
+ 'step_order_ids': [
+ (0, 0, {'sequence': 10, 'step_id': cls.restore_config_step.id}),
+ ],
+ })
+
+ def test_restore(self):
+ # setup master branch
+ master_batch = self.master_bundle._force()
+ with mute_logger('odoo.addons.runbot.models.batch'):
+ master_batch._prepare()
+ reference_slot = master_batch.slot_ids
+ trigger = reference_slot.trigger_id
+ self.assertEqual(trigger.name, 'Server trigger', 'Just checking that we have a single slot')
+ reference_build = reference_slot.build_id
+ self.env['runbot.database'].create({
+ 'build_id': reference_build.id,
+ 'name': f'{reference_build.dest}-suffix',
+ })
+ reference_build.local_state = 'done'
+ reference_build.local_result = 'ok'
+
+ # custom trigger
+ config_data = {
+ 'dump_trigger_id': trigger.id,
+ 'dump_suffix': 'suffix',
+ }
+ self.env['runbot.bundle.trigger.custom'].create({
+ 'bundle_id': self.dev_bundle.id,
+ 'config_id': self.restore_config.id,
+ 'trigger_id': trigger.id,
+ 'config_data': config_data,
+ })
+
+ # create dev build
+ dev_batch = self.dev_bundle._force()
+ with mute_logger('odoo.addons.runbot.models.batch'):
+ dev_batch._prepare()
+ dev_batch.base_reference_batch_id = master_batch # not tested, this is not the purpose of this test
+ dev_build = dev_batch.slot_ids.build_id
+ self.assertEqual(dev_build.params_id.config_data, config_data)
+
+ docker_params = self.restore_config_step._run_restore(dev_build, '/tmp/logs')
+ cmds = docker_params['cmd'].split(' && ')
+ self.assertEqual(f'wget https://False/runbot/static/build/{reference_build.dest}/logs/{reference_build.dest}-suffix.zip', cmds[2])
+ self.assertEqual(f'psql -q {dev_build.dest}-suffix < dump.sql', cmds[8])
+ self.called=True
+
+
class TestBuildConfigStepCreate(TestBuildConfigStepCommon):
diff --git a/runbot/views/bundle_views.xml b/runbot/views/bundle_views.xml
index dd60ca96..acc92635 100644
--- a/runbot/views/bundle_views.xml
+++ b/runbot/views/bundle_views.xml
@@ -36,6 +36,10 @@
runbot.bundle