[IMP] runbot: add auto restore for custom trigger (#784)

One of the most common custom trigger is to restore a build before
starting some test, either to create a multibuild or make the execution
and debug of some test faster.

It is somethimes tedious to use because we need to give an url of a
build to restore. This build must correspond to the right commits,
must still exixt, ... this means that the dump url must be adapted
everytime a branch is rebased.

The way the dump_url is defined is by going on the last batch, following
the link to the `base_reference_batch_id`, finding a slot corresponding
to the right repo set, (ex: Custom enterprise -> enterprise), and
copying the dump_url in this build.

The base_reference_batch_id is eay to automated but we have to find the
right trigger, this is now a parameter of the custom trigger wizard.
There are actually 2 strategy now to define how to download the dump:
- `url`, using `restore_ dump_url`
- `auto`,  using `restore_trigger_id` and `restore_database_suffix`

To ease the setup, a `restore_trigger_id` is added on a trigger, so that
when selecting a trigger, lets say `Custom enterprise`, the defined
`trigger.restore_trigger_id` is automatically chosen for the
`custom_trigger.restore_trigger_id` and the `restore_mode` is setted to
auto.

Two actions are also added to the header of a bundle, a shorcut to
setup a multi build (restore in children) or a restore and test build
(restore in parent).
This commit is contained in:
xdo 2023-06-27 14:13:12 +02:00 committed by GitHub
parent 465081e9f3
commit 24d35988a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 329 additions and 82 deletions

View File

@ -741,17 +741,36 @@ class ConfigStep(models.Model):
if 'dump_url' in params.config_data:
dump_url = params.config_data['dump_url']
zip_name = dump_url.split('/')[-1]
build._log('test-migration', 'Restoring db [%s](%s)' % (zip_name, dump_url), log_type='markdown')
build._log('_run_restore', f'Restoring db [{zip_name}]({dump_url})', log_type='markdown')
suffix = 'all'
else:
download_db_suffix = params.dump_db.db_suffix or self.restore_download_db_suffix
dump_build = params.dump_db.build_id or build.parent_id
if 'dump_trigger_id' in params.config_data:
dump_trigger = self.env['runbot.trigger'].browse(params.config_data['dump_trigger_id'])
dump_suffix = params.config_data.get('dump_suffix', 'all')
base_batch = build.params_id.create_batch_id.base_reference_batch_id
reference_build = base_batch.slot_ids.filtered(lambda s: s.trigger_id == dump_trigger).mapped('build_id')
if not reference_build:
build._log('_run_restore', f'No reference build found in batch {base_batch.id} for trigger {dump_trigger.name}', log_type='markdown', level='ERROR')
build._kill(result='ko')
return
if reference_build.local_state not in ('done', 'running'):
build._log('_run_restore', f'Reference build [{reference_build.id}]({reference_build.build_url} is not yet finished, database may not exist', log_type='markdown', level='WARNING')
dump_db = reference_build.database_ids.filtered(lambda d: d.db_suffix == dump_suffix)
if not dump_db:
build._log('_run_restore', f'No dump with suffix {dump_suffix} found in build [{reference_build.id}]({reference_build.build_url})', log_type='markdown', level='ERROR')
build._kill(result='ko')
return
else:
dump_db = params.dump_db
download_db_suffix = dump_db.db_suffix or self.restore_download_db_suffix
dump_build = dump_db.build_id or build.parent_id
assert download_db_suffix and dump_build
download_db_name = '%s-%s' % (dump_build.dest, download_db_suffix)
zip_name = '%s.zip' % download_db_name
dump_url = '%s%s' % (dump_build.http_log_url(), zip_name)
build._log('test-migration', 'Restoring dump [%s](%s) from build [%s](%s)' % (zip_name, dump_url, dump_build.id, dump_build.build_url), log_type='markdown')
restore_suffix = self.restore_rename_db_suffix or params.dump_db.db_suffix or suffix
restore_suffix = self.restore_rename_db_suffix or dump_db.db_suffix or suffix
assert restore_suffix
restore_db_name = '%s-%s' % (build.dest, restore_suffix)

View File

@ -248,3 +248,35 @@ class Bundle(models.Model):
for branch in self.branch_ids.sorted(key=lambda b: (b.is_pr)):
branch_groups[branch.remote_id.repo_id].append(branch)
return branch_groups
def generate_custom_trigger_multi_action(self):
context = {
'default_bundle_id': self.id,
'default_config_id': self.env.ref('runbot.runbot_build_config_custom_multi').id,
'default_child_config_id': self.env.ref('runbot.runbot_build_config_restore_and_test').id,
'default_extra_params': False,
'default_child_extra_params': '--test-tags /module.test_method',
'default_number_build': 10,
}
return self._generate_custom_trigger_action(context)
def generate_custom_trigger_restore_action(self):
context = {
'default_bundle_id': self.id,
'default_config_id': self.env.ref('runbot.runbot_build_config_restore_and_test').id,
'default_child_config_id': False,
'default_extra_params': '--test-tags /module.test_method',
'default_child_extra_params': False,
'default_number_build': 0,
}
return self._generate_custom_trigger_action(context)
def _generate_custom_trigger_action(self, context):
return {
'type': 'ir.actions.act_window',
'name': 'Generate custom trigger',
'view_mode': 'form',
'res_model': 'runbot.trigger.custom.wizard',
'target': 'new',
'context': context,
}

View File

@ -26,57 +26,135 @@ class CustomTriggerWizard(models.TransientModel):
_name = 'runbot.trigger.custom.wizard'
_description = 'Custom trigger Wizard'
# minimal config options
bundle_id = fields.Many2one('runbot.bundle', "Bundle")
project_id = fields.Many2one(related='bundle_id.project_id', string='Project')
trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', project_id)]")
config_id = fields.Many2one('runbot.build.config', string="Config id", default=lambda self: self.env.ref('runbot.runbot_build_config_custom_multi'))
# base options
config_data = JsonDictField("Config data")
extra_params = fields.Char('Extra params', default='')
# restore options
restore_mode = fields.Selection([('auto', 'Auto'), ('url', 'Dump url')])
restore_dump_url = fields.Char('Dump url for children')
restore_trigger_id = fields.Many2one('runbot.trigger', 'Trigger to restore a dump', domain="[('project_id', '=', project_id), ('manual', '=', False)]")
restore_database_suffix = fields.Char('Database suffix to restore', default='all')
# create multi options
number_build = fields.Integer('Number builds for config multi', default=10)
child_extra_params = fields.Char('Extra params for children', default='--test-tags /module.test_method')
child_dump_url = fields.Char('Dump url for children')
child_config_id = fields.Many2one('runbot.build.config', 'Config for children', default=lambda self: self.env.ref('runbot.runbot_build_config_restore_and_test'))
warnings = fields.Text('Warnings', readonly=True)
@api.onchange('child_extra_params', 'child_dump_url', 'child_config_id', 'number_build', 'config_id', 'trigger_id')
has_create_step = fields.Boolean("Hase create step", compute="_compute_has_create_step")
has_restore_step = fields.Boolean("Hase restore step", compute="_compute_has_restore_step")
has_child_with_restore_step = fields.Boolean("Child config has create step", compute="_compute_has_child_with_restore_step")
@api.depends('config_id')
def _compute_has_create_step(self):
for record in self:
record.has_create_step = any(step.job_type == 'create_build' for step in self.config_id.step_ids())
@api.depends('config_id')
def _compute_has_restore_step(self):
for record in self:
record.has_restore_step = any(step.job_type == 'restore' for step in self.config_id.step_ids())
@api.depends('child_config_id')
def _compute_has_child_with_restore_step(self):
for record in self:
record.has_child_with_restore_step = record.child_config_id and any(step.job_type == 'restore' for step in self.child_config_id.step_ids())
@api.onchange('child_extra_params', 'restore_dump_url', 'config_id', 'child_config_id', 'number_build', 'config_id', 'restore_mode', 'restore_database_suffix', 'restore_trigger_id')
def _onchange_warnings(self):
for wizard in self:
_warnings = []
if not wizard.trigger_id:
_warnings.append(f'No trigger id given (required and may automatically fix other issues)')
if wizard._get_existing_trigger():
_warnings.append(f'A custom trigger already exists for trigger {wizard.trigger_id.name} and will be unlinked')
if wizard.child_dump_url or wizard.child_extra_params or wizard.child_config_id or wizard.number_build:
if not any(step.job_type == 'create_build' for step in wizard.config_id.step_ids()):
_warnings.append('Some multi builds params are given but config as no create step')
if wizard.restore_mode:
if (not wizard.has_restore_step and not wizard.has_child_with_restore_step):
_warnings.append('A restore mode is defined but no config has a restore step')
elif not wizard.restore_mode:
if wizard.has_restore_step :
_warnings.append('Config has a restore step but no restore mode is given')
if wizard.has_child_with_restore_step:
_warnings.append('Child config has a restore step but no restore mode is given')
elif wizard.restore_mode == "url":
if not wizard.restore_dump_url:
_warnings.append('The restore mode is url but no dump_url is given')
elif wizard.restore_mode == "auto":
if not wizard.restore_trigger_id:
_warnings.append('The restore mode is auto but no restore trigger is given')
if not wizard.restore_database_suffix:
_warnings.append('The restore mode is auto but no db suffix is given')
if wizard.child_dump_url and not any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()):
_warnings.append('A dump_url is defined but child config has no restore step')
if not wizard.child_dump_url and any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()):
_warnings.append('Child config has a restore step but no dump_url is given')
if wizard.has_create_step:
if not wizard.child_config_id:
_warnings.append('Config has a create step nut no child config given')
if not wizard.child_extra_params:
_warnings.append('Config has a create step nut no child extra param given')
if wizard.extra_params:
_warnings.append('You may change `Extra params` to `Extra params for children`')
else:
if wizard.child_extra_params:
_warnings.append('Extra params for children given but config has no create step')
if wizard.child_config_id:
_warnings.append('Config for children given but config has no create step')
if not wizard.extra_params:
_warnings.append('No extra params are given')
if not wizard.trigger_id.manual:
_warnings.append("This custom trigger will replace an existing non manual trigger. The ci won't be sent anymore")
wizard.warnings = '\n'.join(_warnings)
@api.onchange('number_build', 'child_extra_params', 'child_dump_url', 'child_config_id')
def _onchange_config_data(self):
@api.onchange('trigger_id')
def _onchange_trigger_id(self):
for wizard in self:
wizard.config_data = self._get_config_data()
if wizard.trigger_id:
wizard.restore_trigger_id = wizard.trigger_id.restore_trigger_id
if wizard.restore_trigger_id and not wizard.restore_mode:
wizard.restore_mode = 'auto'
self._onchange_config_data()
self._onchange_warnings()
@api.onchange('number_build', 'child_extra_params', 'restore_dump_url', 'child_config_id', 'restore_trigger_id', 'restore_database_suffix', 'restore_mode')
def _onchange_config_data(self):
for wizard in self:
wizard.config_data = self._get_config_data()
def _get_config_data(self):
config_data = {}
if self.number_build:
config_data['number_build'] = self.number_build
if self.extra_params:
config_data['extra_params'] = self.extra_params
child_data = {}
if self.child_extra_params:
child_data['extra_params'] = self.child_extra_params
if self.child_dump_url:
child_data['config_data'] = {'dump_url': self.child_dump_url}
if self.restore_mode:
restore_params = {}
if self.restore_mode == 'url':
if self.restore_dump_url:
restore_params['dump_url'] = self.restore_dump_url
else:
if self.restore_trigger_id:
restore_params['dump_trigger_id'] = self.restore_trigger_id.id
if self.restore_database_suffix:
restore_params['dump_suffix'] = self.restore_database_suffix
if self.has_child_with_restore_step:
child_data['config_data'] = restore_params
if not self.has_child_with_restore_step or self.has_restore_step:
config_data.update(restore_params)
if self.child_config_id:
child_data['config_id'] = self.child_config_id.id
if child_data:

View File

@ -38,22 +38,23 @@ class Trigger(models.Model):
sequence = fields.Integer('Sequence')
name = fields.Char("Name")
description = fields.Char("Description", help="Informative description")
project_id = fields.Many2one('runbot.project', string="Project id", required=True, default=lambda self: self.env.ref('runbot.main_project', raise_if_not_found=False))
project_id = fields.Many2one('runbot.project', string="Project id", required=True)
repo_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_triggers', string="Triggers", domain="[('project_id', '=', project_id)]")
dependency_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_dependencies', string="Dependencies")
config_id = fields.Many2one('runbot.build.config', string="Config", required=True)
batch_dependent = fields.Boolean('Batch Dependent', help="Force adding batch in build parameters to make it unique and give access to bundle")
ci_context = fields.Char("Ci context", default='ci/runbot', tracking=True)
ci_context = fields.Char("CI context", tracking=True)
category_id = fields.Many2one('runbot.category', default=lambda self: self.env.ref('runbot.default_category', raise_if_not_found=False))
version_domain = fields.Char(string="Version domain")
hide = fields.Boolean('Hide trigger on main page')
manual = fields.Boolean('Only start trigger manually', default=False)
restore_trigger_id = fields.Many2one('runbot.trigger', string='Restore Trigger ID for custom triggers', help="Mainly usefull to automatically define where to find a reference database when creating a custom trigger", tracking=True)
upgrade_dumps_trigger_id = fields.Many2one('runbot.trigger', string='Template/complement trigger', tracking=True)
upgrade_step_id = fields.Many2one('runbot.build.config.step', compute="_compute_upgrade_step_id", store=True)
ci_url = fields.Char("ci url")
ci_description = fields.Char("ci description")
ci_url = fields.Char("CI url")
ci_description = fields.Char("CI description")
has_stats = fields.Boolean('Has a make_stats config step', compute="_compute_has_stats", store=True)
team_ids = fields.Many2many('runbot.team', string="Runbot Teams", help="Teams responsible of this trigger, mainly usefull for nightly")
@ -276,8 +277,7 @@ class Repo(models.Model):
main_remote_id = fields.Many2one('runbot.remote', "Main remote", tracking=True)
remote_ids = fields.One2many('runbot.remote', 'repo_id', "Remotes")
project_id = fields.Many2one('runbot.project', required=True, tracking=True,
help="Default bundle project to use when pushing on this repos",
default=lambda self: self.env.ref('runbot.main_project', raise_if_not_found=False))
help="Default bundle project to use when pushing on this repos")
# -> not verry usefull, remove it? (iterate on projects or contraints triggers:
# all trigger where a repo is used must be in the same project.
modules = fields.Char("Modules to install", help="Comma-separated list of modules to install and test.", tracking=True)

View File

@ -114,7 +114,7 @@ class RunbotCase(TransactionCase):
'is_pr': False,
'head': self.initial_server_commit.id,
})
self.branch_server.bundle_id # compute
self.master_bundle = self.branch_server.bundle_id # compute
self.dev_bundle = self.Bundle.create({
'name': 'master-dev-tri',
'project_id': self.project.id

View File

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
from unittest.mock import patch, mock_open
from odoo import Command
from odoo.tools import mute_logger
from odoo.exceptions import UserError
from odoo.addons.runbot.common import RunbotException
from .common import RunbotCase
@ -198,6 +199,65 @@ class TestCodeowner(TestBuildConfigStepCommon):
'Requesting review for pull request [base/server:1234](https://example.com/base/server/pull/1234): codeowner-team, team_01, team_02, team_js, team_py'
])
class TestBuildConfigStepRestore(TestBuildConfigStepCommon):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.restore_config_step = cls.env['runbot.build.config.step'].create({
'name': 'restore',
'job_type': 'restore',
})
cls.restore_config = cls.env['runbot.build.config'].create({
'name': 'Restore',
'step_order_ids': [
(0, 0, {'sequence': 10, 'step_id': cls.restore_config_step.id}),
],
})
def test_restore(self):
# setup master branch
master_batch = self.master_bundle._force()
with mute_logger('odoo.addons.runbot.models.batch'):
master_batch._prepare()
reference_slot = master_batch.slot_ids
trigger = reference_slot.trigger_id
self.assertEqual(trigger.name, 'Server trigger', 'Just checking that we have a single slot')
reference_build = reference_slot.build_id
self.env['runbot.database'].create({
'build_id': reference_build.id,
'name': f'{reference_build.dest}-suffix',
})
reference_build.local_state = 'done'
reference_build.local_result = 'ok'
# custom trigger
config_data = {
'dump_trigger_id': trigger.id,
'dump_suffix': 'suffix',
}
self.env['runbot.bundle.trigger.custom'].create({
'bundle_id': self.dev_bundle.id,
'config_id': self.restore_config.id,
'trigger_id': trigger.id,
'config_data': config_data,
})
# create dev build
dev_batch = self.dev_bundle._force()
with mute_logger('odoo.addons.runbot.models.batch'):
dev_batch._prepare()
dev_batch.base_reference_batch_id = master_batch # not tested, this is not the purpose of this test
dev_build = dev_batch.slot_ids.build_id
self.assertEqual(dev_build.params_id.config_data, config_data)
docker_params = self.restore_config_step._run_restore(dev_build, '/tmp/logs')
cmds = docker_params['cmd'].split(' && ')
self.assertEqual(f'wget https://False/runbot/static/build/{reference_build.dest}/logs/{reference_build.dest}-suffix.zip', cmds[2])
self.assertEqual(f'psql -q {dev_build.dest}-suffix < dump.sql', cmds[8])
self.called=True
class TestBuildConfigStepCreate(TestBuildConfigStepCommon):

View File

@ -36,6 +36,10 @@
<field name="model">runbot.bundle</field>
<field name="arch" type="xml">
<form string="Bundles">
<header>
<button name="generate_custom_trigger_multi_action" string="New custom multi" type="object" class="oe_highlight"/>
<button name="generate_custom_trigger_restore_action" string="New custom restore" type="object" class="oe_highlight"/>
</header>
<sheet>
<group>
<group string="Base options">

View File

@ -5,17 +5,29 @@
<field name="model">runbot.trigger.custom.wizard</field>
<field name="arch" type="xml">
<form string="Custom trigger wizard">
<field name="bundle_id" invisible="1"/>
<field name="project_id" invisible="1"/>
<group>
<field name="bundle_id" invisible="1"/>
<field name="project_id" invisible="1"/>
<field name="warnings" decoration-warning="warnings"/>
<field name="trigger_id"/>
<field name="config_id"/>
<field name="number_build"/>
<field name="child_extra_params"/>
<field name="child_dump_url"/>
<field name="child_config_id"/>
<field name="config_data"/>
<group colspan="4">
<field name="warnings" decoration-warning="warnings"/>
</group>
<group string="Base options">
<field name="trigger_id"/>
<field name="config_id"/>
<field name="number_build"/>
<field name="extra_params"/>
<field name="child_extra_params"/>
<field name="child_config_id"/>
</group>
<group string="Restore options">
<field name="restore_mode"/>
<field name="restore_dump_url" attrs="{'invisible': [('restore_mode', '!=', 'url')]}"/>
<field name="restore_trigger_id" attrs="{'invisible': [('restore_mode', '!=', 'auto')]}"/>
<field name="restore_database_suffix" attrs="{'invisible': [('restore_mode', '!=', 'auto')]}"/>
</group>
<group colspan="4">
<field name="config_data"/>
</group>
</group>
<footer>
<button name="submit" string="Submit" type="object" class="btn-primary"/>

View File

@ -9,29 +9,56 @@
</header>
<sheet>
<widget name="web_ribbon" title="Archived" bg_color="bg-danger" attrs="{'invisible': [('active', '=', True)]}"/>
<group name="repo_group">
<field name="active" invisible="1"/>
<field name="name"/>
<field name="sequence"/>
<field name="description"/>
<field name="category_id" required='1'/>
<field name="project_id"/>
<field name="repo_ids"/>
<field name="dependency_ids"/>
<field name="config_id"/>
<field name="batch_dependent"/>
<field name="version_domain" widget="domain" options="{'model': 'runbot.version', 'in_dialog': True}"/>
<field name="hide"/>
<field name="manual"/>
<field name="upgrade_dumps_trigger_id"/>
<field name="upgrade_step_id"/>
<field name="ci_context"/>
<field name="ci_url"/>
<field name="ci_description"/>
<field name="has_stats"/>
<field name="team_ids"/>
<field name="active" invisible="1"/>
<group name="Base config">
<group>
<field name="name"/>
<field name="sequence"/>
<field name="category_id" required='1'/>
<field name="project_id" default=""/>
<field name="config_id"/>
</group>
<group>
<field name="description"/>
<field name="batch_dependent"/>
<field name="upgrade_dumps_trigger_id"/>
<field name="upgrade_step_id"/>
<field name="version_domain" widget="domain" options="{'model': 'runbot.version', 'in_dialog': True}"/>
</group>
</group>
<group>
<group string="Repositories">
<field name="repo_ids" nolabel="1" colspan="2"/>
</group>
<group string="Dependencies">
<field name="dependency_ids" nolabel="1" colspan="2"/>
</group>
</group>
<group>
<group>
<field name="hide"/>
<field name="manual"/>
<field name="restore_trigger_id"/>
</group>
<group>
<field name="ci_context"/>
<field name="ci_url"/>
<field name="ci_description"/>
</group>
</group>
<group string="Managing Team (nightly failure, manual start, ...)"></group>
<field name="team_ids">
<tree>
<field name="name"/>
<field name="github_team"/>
<field name="user_ids" widget="many2many_tags"/>
</tree>
</field>
</sheet>
<div class="oe_chatter">
<field name="message_follower_ids"/>
<field name="message_ids"/>
</div>
</form>
</field>
</record>
@ -77,30 +104,45 @@
<header>
</header>
<sheet>
<group name="repo">
<field name="name"/>
<field name="identity_file"/>
<field name="sequence"/>
<field name="project_id"/>
<field name="modules"/>
<field name="server_files"/>
<field name="manifest_files"/>
<field name="addons_paths"/>
<field name="hook_time" groups="base.group_no_one"/>
<field name="mode"/>
<field name="forbidden_regex"/>
<field name="invalid_branch_message"/>
<field name="single_version"/>
<field name="remote_ids">
<tree string="Remotes" editable="bottom">
<field name="name"/>
<field name="sequence"/>
<field name="fetch_heads" string="Branch"/>
<field name="fetch_pull" string="PR"/>
<field name="send_status"/>
<field name="token" password="True"/>
</tree>
</field>
<group string="Base values">
<group>
<field name="name"/>
<field name="project_id"/>
<field name="sequence"/>
<field name="mode"/>
</group>
<group>
<field name="modules"/>
<field name="server_files"/>
<field name="manifest_files"/>
<field name="addons_paths"/>
</group>
<group colspan="4">
<field name="remote_ids">
<tree string="Remotes" editable="bottom">
<field name="name"/>
<field name="sequence"/>
<field name="fetch_heads" string="Branch"/>
<field name="fetch_pull" string="PR"/>
<field name="send_status"/>
<field name="token" password="True"/>
</tree>
</field>
</group>
</group>
<group string="Advanced options">
<group>
<field name="forbidden_regex"/>
<field name="invalid_branch_message"/>
</group>
<group>
<field name="identity_file"/>
<field name="single_version"/>
</group>
<group string="debug">
<field name="hook_time" groups="base.group_no_one"/>
</group>
</group>
</sheet>
<div class="oe_chatter">