diff --git a/runbot/__manifest__.py b/runbot/__manifest__.py
index 4b5acb53..c7b63641 100644
--- a/runbot/__manifest__.py
+++ b/runbot/__manifest__.py
@@ -52,8 +52,7 @@
'views/stat_views.xml',
'views/upgrade.xml',
'views/warning_views.xml',
-
- 'wizards/mutli_build_wizard_views.xml',
+ 'views/custom_trigger_wizard_views.xml',
'wizards/stat_regex_wizard_views.xml',
],
'license': 'LGPL-3',
diff --git a/runbot/data/runbot_build_config_data.xml b/runbot/data/runbot_build_config_data.xml
index 1d766233..e320c92b 100644
--- a/runbot/data/runbot_build_config_data.xml
+++ b/runbot/data/runbot_build_config_data.xml
@@ -115,5 +115,46 @@
+
+
+ restore
+ restore
+ 2
+
+
+
+ test_only
+ all
+
+ -*
+
+
+ 30
+
+
+
+ Restore and Test
+
+
+
+
+
+
+ custom_create_multi
+ create_build
+
+ 1
+
+
+
+
+ Custom Multi
+ Generic multibuild to use with custom trigger wizard
+
+
+
+
diff --git a/runbot/models/__init__.py b/runbot/models/__init__.py
index 3850b732..46108428 100644
--- a/runbot/models/__init__.py
+++ b/runbot/models/__init__.py
@@ -8,6 +8,7 @@ from . import build_error
from . import bundle
from . import codeowner
from . import commit
+from . import custom_trigger
from . import database
from . import dockerfile
from . import event
diff --git a/runbot/models/batch.py b/runbot/models/batch.py
index 151acea0..3d48e985 100644
--- a/runbot/models/batch.py
+++ b/runbot/models/batch.py
@@ -25,6 +25,7 @@ class Batch(models.Model):
category_id = fields.Many2one('runbot.category', default=lambda self: self.env.ref('runbot.default_category', raise_if_not_found=False))
log_ids = fields.One2many('runbot.batch.log', 'batch_id')
has_warning = fields.Boolean("Has warning")
+ base_reference_batch_id = fields.Many2one('runbot.batch')
@api.depends('slot_ids.build_id')
def _compute_all_build_ids(self):
@@ -216,13 +217,12 @@ class Batch(models.Model):
self._update_commits_infos(base_head_per_repo) # set base_commit, diff infos, ...
# 2. FIND missing commit in a compatible base bundle
- if missing_repos and not bundle.is_base:
+ if not bundle.is_base:
merge_base_commits = self.commit_link_ids.mapped('merge_base_commit_id')
if auto_rebase:
- batch = last_base_batch
- self._log('Using last done batch %s to define missing commits (automatic rebase)', batch.id)
+ self.base_reference_batch_id = last_base_batch
else:
- batch = False
+ self.base_reference_batch_id = False
link_commit = self.env['runbot.commit.link'].search([
('commit_id', 'in', merge_base_commits.ids),
('match_type', 'in', ('new', 'head'))
@@ -234,16 +234,22 @@ class Batch(models.Model):
('category_id', '=', self.category_id.id)
]).sorted(lambda b: (len(b.commit_ids & merge_base_commits), b.id), reverse=True)
if batches:
- batch = batches[0]
- self._log('Using batch [%s](%s) to define missing commits', batch.id, batch._url())
- batch_exiting_commit = batch.commit_ids.filtered(lambda c: c.repo_id in merge_base_commits.repo_id)
- not_matching = (batch_exiting_commit - merge_base_commits)
- if not_matching:
- message = 'Only %s out of %s merge base matched. You may want to rebase your branches to ensure compatibility' % (len(merge_base_commits)-len(not_matching), len(merge_base_commits))
- suggestions = [('Tip: rebase %s to %s' % (commit.repo_id.name, commit.name)) for commit in not_matching]
- self.warning('%s\n%s' % (message, '\n'.join(suggestions)))
+ self.base_reference_batch_id = batches[0]
+
+ batch = self.base_reference_batch_id
if batch:
- fill_missing({link.branch_id: link.commit_id for link in batch.commit_link_ids}, 'base_match')
+ if missing_repos:
+ self._log('Using batch [%s](%s) to define missing commits', batch.id, batch._url())
+ fill_missing({link.branch_id: link.commit_id for link in batch.commit_link_ids}, 'base_match')
+ # check if all mergebase match reference batch
+ batch_exiting_commit = batch.commit_ids.filtered(lambda c: c.repo_id in merge_base_commits.repo_id)
+ not_matching = (batch_exiting_commit - merge_base_commits)
+ if not_matching:
+ message = 'Only %s out of %s merge base matched. You may want to rebase your branches to ensure compatibility' % (len(merge_base_commits)-len(not_matching), len(merge_base_commits))
+ suggestions = [('Tip: rebase %s to %s' % (commit.repo_id.name, commit.name)) for commit in not_matching]
+ self.warning('%s\n%s' % (message, '\n'.join(suggestions)))
+ else:
+ self._log('No reference batch found to fill missing commits')
# 3.1 FIND missing commit in base heads
if missing_repos:
@@ -283,32 +289,33 @@ class Batch(models.Model):
bundle_repos = bundle.branch_ids.mapped('remote_id.repo_id')
version_id = self.bundle_id.version_id.id
project_id = self.bundle_id.project_id.id
- config_by_trigger = {}
- params_by_trigger = {}
+ trigger_customs = {}
for trigger_custom in self.bundle_id.trigger_custom_ids:
- config_by_trigger[trigger_custom.trigger_id.id] = trigger_custom.config_id
- params_by_trigger[trigger_custom.trigger_id.id] = trigger_custom.extra_params
+ trigger_customs[trigger_custom.trigger_id] = trigger_custom
for trigger in triggers:
+ trigger_custom = trigger_customs.get(trigger)
trigger_repos = trigger.repo_ids | trigger.dependency_ids
if trigger_repos & missing_repos:
self.warning('Missing commit for repo %s for trigger %s', (trigger_repos & missing_repos).mapped('name'), trigger.name)
continue
# in any case, search for an existing build
- config = config_by_trigger.get(trigger.id, trigger.config_id)
+ config = trigger_custom.config_id if trigger_custom else trigger.config_id
if not config:
continue
- extra_params = params_by_trigger.get(trigger.id, '')
+ extra_params = trigger_custom.extra_params if trigger_custom else ''
+ config_data = trigger_custom.config_data if trigger_custom else {}
params_value = {
'version_id': version_id,
'extra_params': extra_params,
'config_id': config.id,
'project_id': project_id,
'trigger_id': trigger.id, # for future reference and access rights
- 'config_data': {},
+ 'config_data': config_data,
'commit_link_ids': [(6, 0, [commit_link_by_repos[repo.id].id for repo in trigger_repos])],
'modules': bundle.modules,
'dockerfile_id': dockerfile_id,
'create_batch_id': self.id,
+ 'used_custom_trigger': bool(trigger_custom),
}
params_value['builds_reference_ids'] = trigger._reference_builds(bundle)
diff --git a/runbot/models/build.py b/runbot/models/build.py
index b8c37c4e..da2b8955 100644
--- a/runbot/models/build.py
+++ b/runbot/models/build.py
@@ -58,6 +58,7 @@ class BuildParameters(models.Model):
config_id = fields.Many2one('runbot.build.config', 'Run Config', required=True,
default=lambda self: self.env.ref('runbot.runbot_build_config_default', raise_if_not_found=False), index=True)
config_data = JsonDictField('Config Data')
+ used_custom_trigger = fields.Boolean('Custom trigger was used to generate this build')
build_ids = fields.One2many('runbot.build', 'params_id')
builds_reference_ids = fields.Many2many('runbot.build', relation='runbot_build_params_references', copy=True)
@@ -93,7 +94,10 @@ class BuildParameters(models.Model):
'skip_requirements': param.skip_requirements,
}
if param.trigger_id.batch_dependent:
- cleaned_vals['create_batch_id'] = param.create_batch_id.id,
+ cleaned_vals['create_batch_id'] = param.create_batch_id.id
+ if param.used_custom_trigger:
+ cleaned_vals['used_custom_trigger'] = True
+
param.fingerprint = hashlib.sha256(str(cleaned_vals).encode('utf8')).hexdigest()
@api.depends('commit_link_ids')
@@ -1144,8 +1148,16 @@ class BuildResult(models.Model):
_logger.info('Skipping result for orphan build %s', self.id)
else:
build.parent_id._github_status(post_commit)
- elif build.params_id.config_id == build.params_id.trigger_id.config_id:
- if build.global_result in ('ko', 'warn'):
+ else:
+ trigger = self.params_id.trigger_id
+ if not trigger.ci_context:
+ continue
+
+ desc = trigger.ci_description or " (runtime %ss)" % (build.job_time,)
+ if build.params_id.used_custom_trigger:
+ state = 'error'
+ desc = "This build used custom config. Remove custom trigger to restore default ci"
+ elif build.global_result in ('ko', 'warn'):
state = 'failure'
elif build.global_state in ('pending', 'testing'):
state = 'pending'
@@ -1158,11 +1170,8 @@ class BuildResult(models.Model):
continue
runbot_domain = self.env['runbot.runbot']._domain()
- trigger = self.params_id.trigger_id
target_url = trigger.ci_url or "http://%s/runbot/build/%s" % (runbot_domain, build.id)
- desc = trigger.ci_description or " (runtime %ss)" % (build.job_time,)
- if trigger.ci_context:
- for build_commit in self.params_id.commit_link_ids:
- commit = build_commit.commit_id
- if 'base_' not in build_commit.match_type and commit.repo_id in trigger.repo_ids:
- commit._github_status(build, trigger.ci_context, state, target_url, desc, post_commit)
+ for build_commit in self.params_id.commit_link_ids:
+ commit = build_commit.commit_id
+ if 'base_' not in build_commit.match_type and commit.repo_id in trigger.repo_ids:
+ commit._github_status(build, trigger.ci_context, state, target_url, desc, post_commit)
diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py
index 39f55e2d..cca943da 100644
--- a/runbot/models/build_config.py
+++ b/runbot/models/build_config.py
@@ -55,7 +55,8 @@ class Config(models.Model):
super(Config, self).unlink()
def step_ids(self):
- self.ensure_one()
+ if self:
+ self.ensure_one()
return [ordered_step.step_id for ordered_step in self.step_order_ids.sorted('sequence')]
def _check_step_ids_order(self):
@@ -250,13 +251,18 @@ class ConfigStep(models.Model):
def _run_create_build(self, build, log_path):
count = 0
- for create_config in self.create_config_ids:
- for _ in range(self.number_builds):
+ config_data = build.params_id.config_data
+ config_ids = config_data.get('create_config_ids', self.create_config_ids)
+ for create_config in config_ids:
+ child_data = {'config_id': create_config.id}
+ if 'child_data' in config_data:
+ child_data.update(config_data['child_data'])
+ for _ in range(config_data.get('number_build', self.number_builds)):
count += 1
if count > 200:
build._logger('Too much build created')
break
- child = build._add_child({'config_id': create_config.id}, orphan=self.make_orphan)
+ child = build._add_child(child_data, orphan=self.make_orphan)
build._log('create_build', 'created with config %s' % create_config.name, log_type='subbuild', path=str(child.id))
def make_python_ctx(self, build):
@@ -677,6 +683,7 @@ class ConfigStep(models.Model):
dump_url = params.config_data['dump_url']
zip_name = dump_url.split('/')[-1]
build._log('test-migration', 'Restoring db [%s](%s)' % (zip_name, dump_url), log_type='markdown')
+ suffix = 'all'
else:
download_db_suffix = params.dump_db.db_suffix or self.restore_download_db_suffix
dump_build = params.dump_db.build_id or build.parent_id
@@ -685,7 +692,7 @@ class ConfigStep(models.Model):
zip_name = '%s.zip' % download_db_name
dump_url = '%s%s' % (dump_build.http_log_url(), zip_name)
build._log('test-migration', 'Restoring dump [%s](%s) from build [%s](%s)' % (zip_name, dump_url, dump_build.id, dump_build.build_url), log_type='markdown')
- restore_suffix = self.restore_rename_db_suffix or params.dump_db.db_suffix
+ restore_suffix = self.restore_rename_db_suffix or params.dump_db.db_suffix or suffix
assert restore_suffix
restore_db_name = '%s-%s' % (build.dest, restore_suffix)
diff --git a/runbot/models/bundle.py b/runbot/models/bundle.py
index 693dd36d..f25cccc8 100644
--- a/runbot/models/bundle.py
+++ b/runbot/models/bundle.py
@@ -240,21 +240,3 @@ class Bundle(models.Model):
for branch in self.branch_ids.sorted(key=lambda b: (b.is_pr)):
branch_groups[branch.remote_id.repo_id].append(branch)
return branch_groups
-
-
-class BundleTriggerCustomisation(models.Model):
- _name = 'runbot.bundle.trigger.custom'
- _description = 'Custom trigger'
-
- trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', bundle_id.project_id)]")
- bundle_id = fields.Many2one('runbot.bundle')
- config_id = fields.Many2one('runbot.build.config')
- extra_params = fields.Char("Custom parameters")
-
- _sql_constraints = [
- (
- "bundle_custom_trigger_unique",
- "unique (bundle_id, trigger_id)",
- "Only one custom trigger per trigger per bundle is allowed",
- )
- ]
diff --git a/runbot/models/custom_trigger.py b/runbot/models/custom_trigger.py
new file mode 100644
index 00000000..6cca1c19
--- /dev/null
+++ b/runbot/models/custom_trigger.py
@@ -0,0 +1,97 @@
+import json
+
+from odoo import models, fields, api
+from ..fields import JsonDictField
+
+class BundleTriggerCustomization(models.Model):
+ _name = 'runbot.bundle.trigger.custom'
+ _description = 'Custom trigger'
+
+ trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', bundle_id.project_id)]")
+ bundle_id = fields.Many2one('runbot.bundle')
+ config_id = fields.Many2one('runbot.build.config')
+ extra_params = fields.Char("Custom parameters")
+ config_data = JsonDictField("Config data")
+
+ _sql_constraints = [
+ (
+ "bundle_custom_trigger_unique",
+ "unique (bundle_id, trigger_id)",
+ "Only one custom trigger per trigger per bundle is allowed",
+ )
+ ]
+
+class CustomTriggerWizard(models.TransientModel):
+ _name = 'runbot.trigger.custom.wizard'
+ _description = 'Custom trigger Wizard'
+
+ bundle_id = fields.Many2one('runbot.bundle', "Bundle")
+ project_id = fields.Many2one(related='bundle_id.project_id', string='Project')
+ trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', project_id)]")
+ config_id = fields.Many2one('runbot.build.config', string="Config id", default=lambda self: self.env.ref('runbot.runbot_build_config_custom_multi'))
+
+ config_data = fields.Text("Config data") # Text, hack to make it editable waiting for json widget
+
+ number_build = fields.Integer('Number builds for config multi', default=10)
+
+ child_extra_params = fields.Char('Extra params for children', default='--test-tags /module.test_method')
+ child_dump_url = fields.Char('Dump url for children')
+ child_config_id = fields.Many2one('runbot.build.config', 'Config for children', default=lambda self: self.env.ref('runbot.runbot_build_config_restore_and_test'))
+
+ warnings = fields.Text('Warnings', readonly=True)
+
+ @api.onchange('child_extra_params', 'child_dump_url', 'child_config_id', 'number_build', 'config_id', 'trigger_id')
+ def _onchange_warnings(self):
+ for wizard in self:
+ _warnings = []
+ if wizard._get_existing_trigger():
+ _warnings.append(f'A custom trigger already exists for trigger {wizard.trigger_id.name} and will be unlinked')
+
+ if wizard.child_dump_url or wizard.child_extra_params or wizard.child_config_id or wizard.number_build:
+ if not any(step.job_type == 'create_build' for step in wizard.config_id.step_ids()):
+ _warnings.append('Some multi builds params are given but config as no create step')
+
+ if wizard.child_dump_url and not any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()):
+ _warnings.append('A dump_url is defined but child config has no restore step')
+
+ if not wizard.child_dump_url and any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()):
+ _warnings.append('Child config has a restore step but no dump_url is given')
+
+ if not wizard.trigger_id.manual:
+ _warnings.append("This custom trigger will replace an existing non manual trigger. The ci won't be sent anymore")
+
+ wizard.warnings = '\n'.join(_warnings)
+
+ @api.onchange('number_build', 'child_extra_params', 'child_dump_url', 'child_config_id')
+ def _onchange_config_data(self):
+ for wizard in self:
+ config_data = self._get_config_data()
+ wizard.config_data = json.dumps(config_data, indent=True)
+
+ def _get_config_data(self):
+ config_data = {}
+ if self.number_build:
+ config_data['number_build'] = self.number_build
+ child_data = {}
+ if self.child_extra_params:
+ child_data['extra_params'] = self.child_extra_params
+ if self.child_dump_url:
+ child_data['config_data'] = {'dump_url': self.child_dump_url}
+ if self.child_config_id:
+ child_data['config_id'] = self.child_config_id.id
+ if child_data:
+ config_data['child_data'] = child_data
+ return config_data
+
+ def _get_existing_trigger(self):
+ return self.env['runbot.bundle.trigger.custom'].search([('bundle_id', '=', self.bundle_id.id), ('trigger_id', '=', self.trigger_id.id)])
+
+ def submit(self):
+ self.ensure_one()
+ self._get_existing_trigger().unlink()
+ self.env['runbot.bundle.trigger.custom'].create({
+ 'bundle_id': self.bundle_id.id,
+ 'trigger_id': self.trigger_id.id,
+ 'config_id': self.config_id.id,
+ 'config_data': json.loads(self.config_data),
+ })
diff --git a/runbot/views/custom_trigger_wizard_views.xml b/runbot/views/custom_trigger_wizard_views.xml
new file mode 100644
index 00000000..5d3b065f
--- /dev/null
+++ b/runbot/views/custom_trigger_wizard_views.xml
@@ -0,0 +1,38 @@
+
+
+
+ runbot_trigger_custom_wizard
+ runbot.trigger.custom.wizard
+
+
+
+
+
+
+ Generate custom trigger
+ runbot.trigger.custom.wizard
+ form
+
+ new
+
+ form
+ {'default_bundle_id': active_id}
+
+
diff --git a/runbot/wizards/__init__.py b/runbot/wizards/__init__.py
index aa69f76c..e1c64da3 100644
--- a/runbot/wizards/__init__.py
+++ b/runbot/wizards/__init__.py
@@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
-from . import multi_build_wizard
from . import stat_regex_wizard
diff --git a/runbot/wizards/multi_build_wizard.py b/runbot/wizards/multi_build_wizard.py
deleted file mode 100644
index e8c52a7d..00000000
--- a/runbot/wizards/multi_build_wizard.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from odoo import fields, models, api
-
-
-class MultiBuildWizard(models.TransientModel):
-
- _name = 'runbot.build.config.multi.wizard'
- _description = "Multi wizard"
-
- base_name = fields.Char('Generic name', required=True)
- prefix = fields.Char('Prefix', help="Leave blank to use login.")
- config_multi_name = fields.Char('Config name')
- step_create_multi_name = fields.Char('Create multi step name')
- config_single_name = fields.Char('Config only name')
- config_single_extra_params = fields.Char('Extra cmd args')
- config_single_test_tags = fields.Char('Test tags', default='')
- config_single_test_enable = fields.Boolean('Enable tests', default=True)
- step_single_name = fields.Char('Only step name')
- number_builds = fields.Integer('Number of multi builds', default=10)
- modules = fields.Char('Modules to install', help="List of module patterns to install, use * to install all available modules, prefix the pattern with dash to remove the module.", default='')
-
- @api.onchange('base_name', 'prefix')
- def _onchange_name(self):
- if self.base_name:
- prefix = self.env.user.login.split('@')[0] if not self.prefix else self.prefix
- self.prefix = prefix
- name = '%s %s' % (prefix, self.base_name.capitalize())
- step_name = name.replace(' ', '_').lower()
-
- self.config_multi_name = '%s Multi' % name
- self.step_create_multi_name = '%s_create_multi' % step_name
- self.config_single_name = '%s Single' % name
- self.step_single_name = '%s_single' % step_name
-
- def generate(self):
- if self.base_name:
- # Create the "only" step and config
- step_single = self.env['runbot.build.config.step'].create({
- 'name': self.step_single_name,
- 'job_type': 'install_odoo',
- 'test_tags': self.config_single_test_tags,
- 'extra_params': self.config_single_extra_params,
- 'test_enable': self.config_single_test_enable,
- 'install_modules': self.modules,
- })
- config_single = self.env['runbot.build.config'].create({'name': self.config_single_name})
-
- self.env['runbot.build.config.step.order'].create({
- 'sequence': 10,
- 'config_id': config_single.id,
- 'step_id': step_single.id
- })
-
- # Create the multiple builds step and config
- step_create_multi = self.env['runbot.build.config.step'].create({
- 'name': self.step_create_multi_name,
- 'job_type': 'create_build',
- 'create_config_ids': [(4, config_single.id)],
- 'number_builds': self.number_builds,
- })
-
- config_multi = self.env['runbot.build.config'].create({'name': self.config_multi_name})
-
- config_multi.group = config_multi
- step_create_multi.group = config_multi
- config_single.group = config_multi
- step_single.group = config_multi
-
- self.env['runbot.build.config.step.order'].create({
- 'sequence': 10,
- 'config_id': config_multi.id,
- 'step_id': step_create_multi.id
- })
diff --git a/runbot/wizards/mutli_build_wizard_views.xml b/runbot/wizards/mutli_build_wizard_views.xml
deleted file mode 100644
index 9741b962..00000000
--- a/runbot/wizards/mutli_build_wizard_views.xml
+++ /dev/null
@@ -1,46 +0,0 @@
-
-
-
- runbot_multi_build_wizard
- runbot.build.config.multi.wizard
-
-
-
-
-
-
- Generate Multi Build Config
- ir.actions.act_window
- runbot.build.config.multi.wizard
- form
-
- new
-
-
-
-
-