[IMP] runbot: custom trigger wizard

The actual way to add a custom multibuild on a dev bundle will be:
- use the multibuild wizard to create 2 configs and 2 steps
- add this config on the bundle
- customize this config to make it fatser by installing/restoring...

The usefull parameters are the number of build, the test tags and
modules to install.

Another usefull operation is to restore a dump instead of installing,
especially for post install test only breaking on full databases.

This commit proposes to replace the multi build wizard with a custom
trigger wizard. The main idea is to have generic config, parametrized by
config_data. The wizard will only help to generate the corresponding
config_data.
This commit is contained in:
Xavier-Do 2022-03-10 14:48:06 +01:00 committed by Christophe Monniez
parent 2337bd8518
commit 4e6ae00f23
12 changed files with 236 additions and 176 deletions

View File

@ -52,8 +52,7 @@
'views/stat_views.xml',
'views/upgrade.xml',
'views/warning_views.xml',
'wizards/mutli_build_wizard_views.xml',
'views/custom_trigger_wizard_views.xml',
'wizards/stat_regex_wizard_views.xml',
],
'license': 'LGPL-3',

View File

@ -115,5 +115,46 @@
<field name="step_order_ids" eval="[(5,0,0), (0, 0, {'step_id': ref('runbot_build_config_step_test_click_all')})]"/>
<field name="protected" eval="True"/>
</record>
<record id="runbot_build_config_step_restore" model="runbot.build.config.step">
<field name="name">restore</field>
<field name="job_type">restore</field>
<field name="default_sequence">2</field>
</record>
<record id="runbot_build_config_step_test_only" model="runbot.build.config.step">
<field name="name">test_only</field>
<field name="custom_db_name">all</field>
<field name="create_db" eval="False"/>
<field name="install_modules">-*</field>
<field name="test_enable" eval="True"/>
<field name="protected" eval="True"/>
<field name="default_sequence">30</field>
</record>
<record id="runbot_build_config_restore_and_test" model="runbot.build.config">
<field name="name">Restore and Test</field>
<field name="step_order_ids" eval="[(5,0,0),
(0, 0, {'step_id': ref('runbot_build_config_step_restore')}),
(0, 0, {'step_id': ref('runbot_build_config_step_test_only')})]"/>
<field name="protected" eval="True"/>
</record>
<!-- Multi build custom-->
<record id="runbot_build_config_step_custom_multi_create" model="runbot.build.config.step">
<field name="name">custom_create_multi</field>
<field name="job_type">create_build</field>
<field name="create_config_ids" eval="[(4, ref('runbot_build_config_restore_and_test'))]"/>
<field name="number_builds">1</field>
<field name="protected" eval="True"/>
</record>
<record id="runbot_build_config_custom_multi" model="runbot.build.config">
<field name="name">Custom Multi</field>
<field name="description">Generic multibuild to use with custom trigger wizard</field>
<field name="step_order_ids" eval="[(5,0,0), (0, 0, {'step_id': ref('runbot_build_config_step_create_light_multi')})]"/>
<field name="protected" eval="True"/>
</record>
</data>
</odoo>

View File

@ -8,6 +8,7 @@ from . import build_error
from . import bundle
from . import codeowner
from . import commit
from . import custom_trigger
from . import database
from . import dockerfile
from . import event

View File

@ -25,6 +25,7 @@ class Batch(models.Model):
category_id = fields.Many2one('runbot.category', default=lambda self: self.env.ref('runbot.default_category', raise_if_not_found=False))
log_ids = fields.One2many('runbot.batch.log', 'batch_id')
has_warning = fields.Boolean("Has warning")
base_reference_batch_id = fields.Many2one('runbot.batch')
@api.depends('slot_ids.build_id')
def _compute_all_build_ids(self):
@ -216,13 +217,12 @@ class Batch(models.Model):
self._update_commits_infos(base_head_per_repo) # set base_commit, diff infos, ...
# 2. FIND missing commit in a compatible base bundle
if missing_repos and not bundle.is_base:
if not bundle.is_base:
merge_base_commits = self.commit_link_ids.mapped('merge_base_commit_id')
if auto_rebase:
batch = last_base_batch
self._log('Using last done batch %s to define missing commits (automatic rebase)', batch.id)
self.base_reference_batch_id = last_base_batch
else:
batch = False
self.base_reference_batch_id = False
link_commit = self.env['runbot.commit.link'].search([
('commit_id', 'in', merge_base_commits.ids),
('match_type', 'in', ('new', 'head'))
@ -234,16 +234,22 @@ class Batch(models.Model):
('category_id', '=', self.category_id.id)
]).sorted(lambda b: (len(b.commit_ids & merge_base_commits), b.id), reverse=True)
if batches:
batch = batches[0]
self._log('Using batch [%s](%s) to define missing commits', batch.id, batch._url())
batch_exiting_commit = batch.commit_ids.filtered(lambda c: c.repo_id in merge_base_commits.repo_id)
not_matching = (batch_exiting_commit - merge_base_commits)
if not_matching:
message = 'Only %s out of %s merge base matched. You may want to rebase your branches to ensure compatibility' % (len(merge_base_commits)-len(not_matching), len(merge_base_commits))
suggestions = [('Tip: rebase %s to %s' % (commit.repo_id.name, commit.name)) for commit in not_matching]
self.warning('%s\n%s' % (message, '\n'.join(suggestions)))
self.base_reference_batch_id = batches[0]
batch = self.base_reference_batch_id
if batch:
fill_missing({link.branch_id: link.commit_id for link in batch.commit_link_ids}, 'base_match')
if missing_repos:
self._log('Using batch [%s](%s) to define missing commits', batch.id, batch._url())
fill_missing({link.branch_id: link.commit_id for link in batch.commit_link_ids}, 'base_match')
# check if all mergebase match reference batch
batch_exiting_commit = batch.commit_ids.filtered(lambda c: c.repo_id in merge_base_commits.repo_id)
not_matching = (batch_exiting_commit - merge_base_commits)
if not_matching:
message = 'Only %s out of %s merge base matched. You may want to rebase your branches to ensure compatibility' % (len(merge_base_commits)-len(not_matching), len(merge_base_commits))
suggestions = [('Tip: rebase %s to %s' % (commit.repo_id.name, commit.name)) for commit in not_matching]
self.warning('%s\n%s' % (message, '\n'.join(suggestions)))
else:
self._log('No reference batch found to fill missing commits')
# 3.1 FIND missing commit in base heads
if missing_repos:
@ -283,32 +289,33 @@ class Batch(models.Model):
bundle_repos = bundle.branch_ids.mapped('remote_id.repo_id')
version_id = self.bundle_id.version_id.id
project_id = self.bundle_id.project_id.id
config_by_trigger = {}
params_by_trigger = {}
trigger_customs = {}
for trigger_custom in self.bundle_id.trigger_custom_ids:
config_by_trigger[trigger_custom.trigger_id.id] = trigger_custom.config_id
params_by_trigger[trigger_custom.trigger_id.id] = trigger_custom.extra_params
trigger_customs[trigger_custom.trigger_id] = trigger_custom
for trigger in triggers:
trigger_custom = trigger_customs.get(trigger)
trigger_repos = trigger.repo_ids | trigger.dependency_ids
if trigger_repos & missing_repos:
self.warning('Missing commit for repo %s for trigger %s', (trigger_repos & missing_repos).mapped('name'), trigger.name)
continue
# in any case, search for an existing build
config = config_by_trigger.get(trigger.id, trigger.config_id)
config = trigger_custom.config_id if trigger_custom else trigger.config_id
if not config:
continue
extra_params = params_by_trigger.get(trigger.id, '')
extra_params = trigger_custom.extra_params if trigger_custom else ''
config_data = trigger_custom.config_data if trigger_custom else {}
params_value = {
'version_id': version_id,
'extra_params': extra_params,
'config_id': config.id,
'project_id': project_id,
'trigger_id': trigger.id, # for future reference and access rights
'config_data': {},
'config_data': config_data,
'commit_link_ids': [(6, 0, [commit_link_by_repos[repo.id].id for repo in trigger_repos])],
'modules': bundle.modules,
'dockerfile_id': dockerfile_id,
'create_batch_id': self.id,
'used_custom_trigger': bool(trigger_custom),
}
params_value['builds_reference_ids'] = trigger._reference_builds(bundle)

View File

@ -58,6 +58,7 @@ class BuildParameters(models.Model):
config_id = fields.Many2one('runbot.build.config', 'Run Config', required=True,
default=lambda self: self.env.ref('runbot.runbot_build_config_default', raise_if_not_found=False), index=True)
config_data = JsonDictField('Config Data')
used_custom_trigger = fields.Boolean('Custom trigger was used to generate this build')
build_ids = fields.One2many('runbot.build', 'params_id')
builds_reference_ids = fields.Many2many('runbot.build', relation='runbot_build_params_references', copy=True)
@ -93,7 +94,10 @@ class BuildParameters(models.Model):
'skip_requirements': param.skip_requirements,
}
if param.trigger_id.batch_dependent:
cleaned_vals['create_batch_id'] = param.create_batch_id.id,
cleaned_vals['create_batch_id'] = param.create_batch_id.id
if param.used_custom_trigger:
cleaned_vals['used_custom_trigger'] = True
param.fingerprint = hashlib.sha256(str(cleaned_vals).encode('utf8')).hexdigest()
@api.depends('commit_link_ids')
@ -1144,8 +1148,16 @@ class BuildResult(models.Model):
_logger.info('Skipping result for orphan build %s', self.id)
else:
build.parent_id._github_status(post_commit)
elif build.params_id.config_id == build.params_id.trigger_id.config_id:
if build.global_result in ('ko', 'warn'):
else:
trigger = self.params_id.trigger_id
if not trigger.ci_context:
continue
desc = trigger.ci_description or " (runtime %ss)" % (build.job_time,)
if build.params_id.used_custom_trigger:
state = 'error'
desc = "This build used custom config. Remove custom trigger to restore default ci"
elif build.global_result in ('ko', 'warn'):
state = 'failure'
elif build.global_state in ('pending', 'testing'):
state = 'pending'
@ -1158,11 +1170,8 @@ class BuildResult(models.Model):
continue
runbot_domain = self.env['runbot.runbot']._domain()
trigger = self.params_id.trigger_id
target_url = trigger.ci_url or "http://%s/runbot/build/%s" % (runbot_domain, build.id)
desc = trigger.ci_description or " (runtime %ss)" % (build.job_time,)
if trigger.ci_context:
for build_commit in self.params_id.commit_link_ids:
commit = build_commit.commit_id
if 'base_' not in build_commit.match_type and commit.repo_id in trigger.repo_ids:
commit._github_status(build, trigger.ci_context, state, target_url, desc, post_commit)
for build_commit in self.params_id.commit_link_ids:
commit = build_commit.commit_id
if 'base_' not in build_commit.match_type and commit.repo_id in trigger.repo_ids:
commit._github_status(build, trigger.ci_context, state, target_url, desc, post_commit)

View File

@ -55,7 +55,8 @@ class Config(models.Model):
super(Config, self).unlink()
def step_ids(self):
self.ensure_one()
if self:
self.ensure_one()
return [ordered_step.step_id for ordered_step in self.step_order_ids.sorted('sequence')]
def _check_step_ids_order(self):
@ -250,13 +251,18 @@ class ConfigStep(models.Model):
def _run_create_build(self, build, log_path):
count = 0
for create_config in self.create_config_ids:
for _ in range(self.number_builds):
config_data = build.params_id.config_data
config_ids = config_data.get('create_config_ids', self.create_config_ids)
for create_config in config_ids:
child_data = {'config_id': create_config.id}
if 'child_data' in config_data:
child_data.update(config_data['child_data'])
for _ in range(config_data.get('number_build', self.number_builds)):
count += 1
if count > 200:
build._logger('Too much build created')
break
child = build._add_child({'config_id': create_config.id}, orphan=self.make_orphan)
child = build._add_child(child_data, orphan=self.make_orphan)
build._log('create_build', 'created with config %s' % create_config.name, log_type='subbuild', path=str(child.id))
def make_python_ctx(self, build):
@ -677,6 +683,7 @@ class ConfigStep(models.Model):
dump_url = params.config_data['dump_url']
zip_name = dump_url.split('/')[-1]
build._log('test-migration', 'Restoring db [%s](%s)' % (zip_name, dump_url), log_type='markdown')
suffix = 'all'
else:
download_db_suffix = params.dump_db.db_suffix or self.restore_download_db_suffix
dump_build = params.dump_db.build_id or build.parent_id
@ -685,7 +692,7 @@ class ConfigStep(models.Model):
zip_name = '%s.zip' % download_db_name
dump_url = '%s%s' % (dump_build.http_log_url(), zip_name)
build._log('test-migration', 'Restoring dump [%s](%s) from build [%s](%s)' % (zip_name, dump_url, dump_build.id, dump_build.build_url), log_type='markdown')
restore_suffix = self.restore_rename_db_suffix or params.dump_db.db_suffix
restore_suffix = self.restore_rename_db_suffix or params.dump_db.db_suffix or suffix
assert restore_suffix
restore_db_name = '%s-%s' % (build.dest, restore_suffix)

View File

@ -240,21 +240,3 @@ class Bundle(models.Model):
for branch in self.branch_ids.sorted(key=lambda b: (b.is_pr)):
branch_groups[branch.remote_id.repo_id].append(branch)
return branch_groups
class BundleTriggerCustomisation(models.Model):
_name = 'runbot.bundle.trigger.custom'
_description = 'Custom trigger'
trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', bundle_id.project_id)]")
bundle_id = fields.Many2one('runbot.bundle')
config_id = fields.Many2one('runbot.build.config')
extra_params = fields.Char("Custom parameters")
_sql_constraints = [
(
"bundle_custom_trigger_unique",
"unique (bundle_id, trigger_id)",
"Only one custom trigger per trigger per bundle is allowed",
)
]

View File

@ -0,0 +1,97 @@
import json
from odoo import models, fields, api
from ..fields import JsonDictField
class BundleTriggerCustomization(models.Model):
_name = 'runbot.bundle.trigger.custom'
_description = 'Custom trigger'
trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', bundle_id.project_id)]")
bundle_id = fields.Many2one('runbot.bundle')
config_id = fields.Many2one('runbot.build.config')
extra_params = fields.Char("Custom parameters")
config_data = JsonDictField("Config data")
_sql_constraints = [
(
"bundle_custom_trigger_unique",
"unique (bundle_id, trigger_id)",
"Only one custom trigger per trigger per bundle is allowed",
)
]
class CustomTriggerWizard(models.TransientModel):
_name = 'runbot.trigger.custom.wizard'
_description = 'Custom trigger Wizard'
bundle_id = fields.Many2one('runbot.bundle', "Bundle")
project_id = fields.Many2one(related='bundle_id.project_id', string='Project')
trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', project_id)]")
config_id = fields.Many2one('runbot.build.config', string="Config id", default=lambda self: self.env.ref('runbot.runbot_build_config_custom_multi'))
config_data = fields.Text("Config data") # Text, hack to make it editable waiting for json widget
number_build = fields.Integer('Number builds for config multi', default=10)
child_extra_params = fields.Char('Extra params for children', default='--test-tags /module.test_method')
child_dump_url = fields.Char('Dump url for children')
child_config_id = fields.Many2one('runbot.build.config', 'Config for children', default=lambda self: self.env.ref('runbot.runbot_build_config_restore_and_test'))
warnings = fields.Text('Warnings', readonly=True)
@api.onchange('child_extra_params', 'child_dump_url', 'child_config_id', 'number_build', 'config_id', 'trigger_id')
def _onchange_warnings(self):
for wizard in self:
_warnings = []
if wizard._get_existing_trigger():
_warnings.append(f'A custom trigger already exists for trigger {wizard.trigger_id.name} and will be unlinked')
if wizard.child_dump_url or wizard.child_extra_params or wizard.child_config_id or wizard.number_build:
if not any(step.job_type == 'create_build' for step in wizard.config_id.step_ids()):
_warnings.append('Some multi builds params are given but config as no create step')
if wizard.child_dump_url and not any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()):
_warnings.append('A dump_url is defined but child config has no restore step')
if not wizard.child_dump_url and any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()):
_warnings.append('Child config has a restore step but no dump_url is given')
if not wizard.trigger_id.manual:
_warnings.append("This custom trigger will replace an existing non manual trigger. The ci won't be sent anymore")
wizard.warnings = '\n'.join(_warnings)
@api.onchange('number_build', 'child_extra_params', 'child_dump_url', 'child_config_id')
def _onchange_config_data(self):
for wizard in self:
config_data = self._get_config_data()
wizard.config_data = json.dumps(config_data, indent=True)
def _get_config_data(self):
config_data = {}
if self.number_build:
config_data['number_build'] = self.number_build
child_data = {}
if self.child_extra_params:
child_data['extra_params'] = self.child_extra_params
if self.child_dump_url:
child_data['config_data'] = {'dump_url': self.child_dump_url}
if self.child_config_id:
child_data['config_id'] = self.child_config_id.id
if child_data:
config_data['child_data'] = child_data
return config_data
def _get_existing_trigger(self):
return self.env['runbot.bundle.trigger.custom'].search([('bundle_id', '=', self.bundle_id.id), ('trigger_id', '=', self.trigger_id.id)])
def submit(self):
self.ensure_one()
self._get_existing_trigger().unlink()
self.env['runbot.bundle.trigger.custom'].create({
'bundle_id': self.bundle_id.id,
'trigger_id': self.trigger_id.id,
'config_id': self.config_id.id,
'config_data': json.loads(self.config_data),
})

View File

@ -0,0 +1,38 @@
<?xml version="1.0" encoding="UTF-8"?>
<odoo>
<record model="ir.ui.view" id="runbot_trigger_custom_wizard_form">
<field name="name">runbot_trigger_custom_wizard</field>
<field name="model">runbot.trigger.custom.wizard</field>
<field name="arch" type="xml">
<form string="Custom trigger wizard">
<group>
<field name="bundle_id" invisible="1"/>
<field name="project_id" invisible="1"/>
<field name="warnings" decoration-warning="warnings"/>
<field name="trigger_id"/>
<field name="config_id"/>
<field name="number_build"/>
<field name="child_extra_params"/>
<field name="child_dump_url"/>
<field name="child_config_id"/>
<field name="config_data"/>
</group>
<footer>
<button name="submit" string="Submit" type="object" class="btn-primary"/>
<button string="Cancel" special="cancel" class="btn-default"/>
</footer>
</form>
</field>
</record>
<record model="ir.actions.act_window" id="runbot_trigger_custom_wizard_action">
<field name="name">Generate custom trigger</field>
<field name="res_model">runbot.trigger.custom.wizard</field>
<field name="view_mode">form</field>
<field name="view_id" ref="runbot_trigger_custom_wizard_form"/>
<field name="target">new</field>
<field name="binding_model_id" ref="runbot.model_runbot_bundle"/>
<field name="binding_view_types">form</field>
<field name="context">{'default_bundle_id': active_id}</field>
</record>
</odoo>

View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
from . import multi_build_wizard
from . import stat_regex_wizard

View File

@ -1,74 +0,0 @@
# -*- coding: utf-8 -*-
from odoo import fields, models, api
class MultiBuildWizard(models.TransientModel):
_name = 'runbot.build.config.multi.wizard'
_description = "Multi wizard"
base_name = fields.Char('Generic name', required=True)
prefix = fields.Char('Prefix', help="Leave blank to use login.")
config_multi_name = fields.Char('Config name')
step_create_multi_name = fields.Char('Create multi step name')
config_single_name = fields.Char('Config only name')
config_single_extra_params = fields.Char('Extra cmd args')
config_single_test_tags = fields.Char('Test tags', default='')
config_single_test_enable = fields.Boolean('Enable tests', default=True)
step_single_name = fields.Char('Only step name')
number_builds = fields.Integer('Number of multi builds', default=10)
modules = fields.Char('Modules to install', help="List of module patterns to install, use * to install all available modules, prefix the pattern with dash to remove the module.", default='')
@api.onchange('base_name', 'prefix')
def _onchange_name(self):
if self.base_name:
prefix = self.env.user.login.split('@')[0] if not self.prefix else self.prefix
self.prefix = prefix
name = '%s %s' % (prefix, self.base_name.capitalize())
step_name = name.replace(' ', '_').lower()
self.config_multi_name = '%s Multi' % name
self.step_create_multi_name = '%s_create_multi' % step_name
self.config_single_name = '%s Single' % name
self.step_single_name = '%s_single' % step_name
def generate(self):
if self.base_name:
# Create the "only" step and config
step_single = self.env['runbot.build.config.step'].create({
'name': self.step_single_name,
'job_type': 'install_odoo',
'test_tags': self.config_single_test_tags,
'extra_params': self.config_single_extra_params,
'test_enable': self.config_single_test_enable,
'install_modules': self.modules,
})
config_single = self.env['runbot.build.config'].create({'name': self.config_single_name})
self.env['runbot.build.config.step.order'].create({
'sequence': 10,
'config_id': config_single.id,
'step_id': step_single.id
})
# Create the multiple builds step and config
step_create_multi = self.env['runbot.build.config.step'].create({
'name': self.step_create_multi_name,
'job_type': 'create_build',
'create_config_ids': [(4, config_single.id)],
'number_builds': self.number_builds,
})
config_multi = self.env['runbot.build.config'].create({'name': self.config_multi_name})
config_multi.group = config_multi
step_create_multi.group = config_multi
config_single.group = config_multi
step_single.group = config_multi
self.env['runbot.build.config.step.order'].create({
'sequence': 10,
'config_id': config_multi.id,
'step_id': step_create_multi.id
})

View File

@ -1,46 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<odoo>
<record model="ir.ui.view" id="runbot_multi_build_wizard_form">
<field name="name">runbot_multi_build_wizard</field>
<field name="model">runbot.build.config.multi.wizard</field>
<field name="arch" type="xml">
<form string="Send Mail">
<group>
<field name="base_name"/>
<field name="prefix"/>
<field name="config_multi_name"/>
<field name="step_create_multi_name"/>
<field name="config_single_name"/>
<field name="step_single_name"/>
<field name="number_builds"/>
<field name="config_single_test_tags"/>
<field name="modules"/>
<field name="config_single_extra_params"/>
<field name="config_single_test_enable"/>
</group>
<footer>
<button name="generate" string="Create" type="object" class="btn-primary"/>
<button string="Cancel" special="cancel" class="btn-default"/>
</footer>
</form>
</field>
</record>
<record model="ir.actions.act_window" id="runbot_multi_build_wizard_action">
<field name="name">Generate Multi Build Config</field>
<field name="type">ir.actions.act_window</field>
<field name="res_model">runbot.build.config.multi.wizard</field>
<field name="view_mode">form</field>
<field name="view_id" ref="runbot_multi_build_wizard_form"/>
<field name="target">new</field>
</record>
<menuitem
name="Multi Build Wizard"
id="runbot_menu_multi_wizard"
parent="runbot_menu_configs"
sequence="30"
action="runbot_multi_build_wizard_action"
/>
</odoo>