diff --git a/runbot/__manifest__.py b/runbot/__manifest__.py index 4b5acb53..c7b63641 100644 --- a/runbot/__manifest__.py +++ b/runbot/__manifest__.py @@ -52,8 +52,7 @@ 'views/stat_views.xml', 'views/upgrade.xml', 'views/warning_views.xml', - - 'wizards/mutli_build_wizard_views.xml', + 'views/custom_trigger_wizard_views.xml', 'wizards/stat_regex_wizard_views.xml', ], 'license': 'LGPL-3', diff --git a/runbot/data/runbot_build_config_data.xml b/runbot/data/runbot_build_config_data.xml index 1d766233..e320c92b 100644 --- a/runbot/data/runbot_build_config_data.xml +++ b/runbot/data/runbot_build_config_data.xml @@ -115,5 +115,46 @@ + + + restore + restore + 2 + + + + test_only + all + + -* + + + 30 + + + + Restore and Test + + + + + + + custom_create_multi + create_build + + 1 + + + + + Custom Multi + Generic multibuild to use with custom trigger wizard + + + + diff --git a/runbot/models/__init__.py b/runbot/models/__init__.py index 3850b732..46108428 100644 --- a/runbot/models/__init__.py +++ b/runbot/models/__init__.py @@ -8,6 +8,7 @@ from . import build_error from . import bundle from . import codeowner from . import commit +from . import custom_trigger from . import database from . import dockerfile from . import event diff --git a/runbot/models/batch.py b/runbot/models/batch.py index 151acea0..3d48e985 100644 --- a/runbot/models/batch.py +++ b/runbot/models/batch.py @@ -25,6 +25,7 @@ class Batch(models.Model): category_id = fields.Many2one('runbot.category', default=lambda self: self.env.ref('runbot.default_category', raise_if_not_found=False)) log_ids = fields.One2many('runbot.batch.log', 'batch_id') has_warning = fields.Boolean("Has warning") + base_reference_batch_id = fields.Many2one('runbot.batch') @api.depends('slot_ids.build_id') def _compute_all_build_ids(self): @@ -216,13 +217,12 @@ class Batch(models.Model): self._update_commits_infos(base_head_per_repo) # set base_commit, diff infos, ... # 2. FIND missing commit in a compatible base bundle - if missing_repos and not bundle.is_base: + if not bundle.is_base: merge_base_commits = self.commit_link_ids.mapped('merge_base_commit_id') if auto_rebase: - batch = last_base_batch - self._log('Using last done batch %s to define missing commits (automatic rebase)', batch.id) + self.base_reference_batch_id = last_base_batch else: - batch = False + self.base_reference_batch_id = False link_commit = self.env['runbot.commit.link'].search([ ('commit_id', 'in', merge_base_commits.ids), ('match_type', 'in', ('new', 'head')) @@ -234,16 +234,22 @@ class Batch(models.Model): ('category_id', '=', self.category_id.id) ]).sorted(lambda b: (len(b.commit_ids & merge_base_commits), b.id), reverse=True) if batches: - batch = batches[0] - self._log('Using batch [%s](%s) to define missing commits', batch.id, batch._url()) - batch_exiting_commit = batch.commit_ids.filtered(lambda c: c.repo_id in merge_base_commits.repo_id) - not_matching = (batch_exiting_commit - merge_base_commits) - if not_matching: - message = 'Only %s out of %s merge base matched. You may want to rebase your branches to ensure compatibility' % (len(merge_base_commits)-len(not_matching), len(merge_base_commits)) - suggestions = [('Tip: rebase %s to %s' % (commit.repo_id.name, commit.name)) for commit in not_matching] - self.warning('%s\n%s' % (message, '\n'.join(suggestions))) + self.base_reference_batch_id = batches[0] + + batch = self.base_reference_batch_id if batch: - fill_missing({link.branch_id: link.commit_id for link in batch.commit_link_ids}, 'base_match') + if missing_repos: + self._log('Using batch [%s](%s) to define missing commits', batch.id, batch._url()) + fill_missing({link.branch_id: link.commit_id for link in batch.commit_link_ids}, 'base_match') + # check if all mergebase match reference batch + batch_exiting_commit = batch.commit_ids.filtered(lambda c: c.repo_id in merge_base_commits.repo_id) + not_matching = (batch_exiting_commit - merge_base_commits) + if not_matching: + message = 'Only %s out of %s merge base matched. You may want to rebase your branches to ensure compatibility' % (len(merge_base_commits)-len(not_matching), len(merge_base_commits)) + suggestions = [('Tip: rebase %s to %s' % (commit.repo_id.name, commit.name)) for commit in not_matching] + self.warning('%s\n%s' % (message, '\n'.join(suggestions))) + else: + self._log('No reference batch found to fill missing commits') # 3.1 FIND missing commit in base heads if missing_repos: @@ -283,32 +289,33 @@ class Batch(models.Model): bundle_repos = bundle.branch_ids.mapped('remote_id.repo_id') version_id = self.bundle_id.version_id.id project_id = self.bundle_id.project_id.id - config_by_trigger = {} - params_by_trigger = {} + trigger_customs = {} for trigger_custom in self.bundle_id.trigger_custom_ids: - config_by_trigger[trigger_custom.trigger_id.id] = trigger_custom.config_id - params_by_trigger[trigger_custom.trigger_id.id] = trigger_custom.extra_params + trigger_customs[trigger_custom.trigger_id] = trigger_custom for trigger in triggers: + trigger_custom = trigger_customs.get(trigger) trigger_repos = trigger.repo_ids | trigger.dependency_ids if trigger_repos & missing_repos: self.warning('Missing commit for repo %s for trigger %s', (trigger_repos & missing_repos).mapped('name'), trigger.name) continue # in any case, search for an existing build - config = config_by_trigger.get(trigger.id, trigger.config_id) + config = trigger_custom.config_id if trigger_custom else trigger.config_id if not config: continue - extra_params = params_by_trigger.get(trigger.id, '') + extra_params = trigger_custom.extra_params if trigger_custom else '' + config_data = trigger_custom.config_data if trigger_custom else {} params_value = { 'version_id': version_id, 'extra_params': extra_params, 'config_id': config.id, 'project_id': project_id, 'trigger_id': trigger.id, # for future reference and access rights - 'config_data': {}, + 'config_data': config_data, 'commit_link_ids': [(6, 0, [commit_link_by_repos[repo.id].id for repo in trigger_repos])], 'modules': bundle.modules, 'dockerfile_id': dockerfile_id, 'create_batch_id': self.id, + 'used_custom_trigger': bool(trigger_custom), } params_value['builds_reference_ids'] = trigger._reference_builds(bundle) diff --git a/runbot/models/build.py b/runbot/models/build.py index b8c37c4e..da2b8955 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -58,6 +58,7 @@ class BuildParameters(models.Model): config_id = fields.Many2one('runbot.build.config', 'Run Config', required=True, default=lambda self: self.env.ref('runbot.runbot_build_config_default', raise_if_not_found=False), index=True) config_data = JsonDictField('Config Data') + used_custom_trigger = fields.Boolean('Custom trigger was used to generate this build') build_ids = fields.One2many('runbot.build', 'params_id') builds_reference_ids = fields.Many2many('runbot.build', relation='runbot_build_params_references', copy=True) @@ -93,7 +94,10 @@ class BuildParameters(models.Model): 'skip_requirements': param.skip_requirements, } if param.trigger_id.batch_dependent: - cleaned_vals['create_batch_id'] = param.create_batch_id.id, + cleaned_vals['create_batch_id'] = param.create_batch_id.id + if param.used_custom_trigger: + cleaned_vals['used_custom_trigger'] = True + param.fingerprint = hashlib.sha256(str(cleaned_vals).encode('utf8')).hexdigest() @api.depends('commit_link_ids') @@ -1144,8 +1148,16 @@ class BuildResult(models.Model): _logger.info('Skipping result for orphan build %s', self.id) else: build.parent_id._github_status(post_commit) - elif build.params_id.config_id == build.params_id.trigger_id.config_id: - if build.global_result in ('ko', 'warn'): + else: + trigger = self.params_id.trigger_id + if not trigger.ci_context: + continue + + desc = trigger.ci_description or " (runtime %ss)" % (build.job_time,) + if build.params_id.used_custom_trigger: + state = 'error' + desc = "This build used custom config. Remove custom trigger to restore default ci" + elif build.global_result in ('ko', 'warn'): state = 'failure' elif build.global_state in ('pending', 'testing'): state = 'pending' @@ -1158,11 +1170,8 @@ class BuildResult(models.Model): continue runbot_domain = self.env['runbot.runbot']._domain() - trigger = self.params_id.trigger_id target_url = trigger.ci_url or "http://%s/runbot/build/%s" % (runbot_domain, build.id) - desc = trigger.ci_description or " (runtime %ss)" % (build.job_time,) - if trigger.ci_context: - for build_commit in self.params_id.commit_link_ids: - commit = build_commit.commit_id - if 'base_' not in build_commit.match_type and commit.repo_id in trigger.repo_ids: - commit._github_status(build, trigger.ci_context, state, target_url, desc, post_commit) + for build_commit in self.params_id.commit_link_ids: + commit = build_commit.commit_id + if 'base_' not in build_commit.match_type and commit.repo_id in trigger.repo_ids: + commit._github_status(build, trigger.ci_context, state, target_url, desc, post_commit) diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index 39f55e2d..cca943da 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -55,7 +55,8 @@ class Config(models.Model): super(Config, self).unlink() def step_ids(self): - self.ensure_one() + if self: + self.ensure_one() return [ordered_step.step_id for ordered_step in self.step_order_ids.sorted('sequence')] def _check_step_ids_order(self): @@ -250,13 +251,18 @@ class ConfigStep(models.Model): def _run_create_build(self, build, log_path): count = 0 - for create_config in self.create_config_ids: - for _ in range(self.number_builds): + config_data = build.params_id.config_data + config_ids = config_data.get('create_config_ids', self.create_config_ids) + for create_config in config_ids: + child_data = {'config_id': create_config.id} + if 'child_data' in config_data: + child_data.update(config_data['child_data']) + for _ in range(config_data.get('number_build', self.number_builds)): count += 1 if count > 200: build._logger('Too much build created') break - child = build._add_child({'config_id': create_config.id}, orphan=self.make_orphan) + child = build._add_child(child_data, orphan=self.make_orphan) build._log('create_build', 'created with config %s' % create_config.name, log_type='subbuild', path=str(child.id)) def make_python_ctx(self, build): @@ -677,6 +683,7 @@ class ConfigStep(models.Model): dump_url = params.config_data['dump_url'] zip_name = dump_url.split('/')[-1] build._log('test-migration', 'Restoring db [%s](%s)' % (zip_name, dump_url), log_type='markdown') + suffix = 'all' else: download_db_suffix = params.dump_db.db_suffix or self.restore_download_db_suffix dump_build = params.dump_db.build_id or build.parent_id @@ -685,7 +692,7 @@ class ConfigStep(models.Model): zip_name = '%s.zip' % download_db_name dump_url = '%s%s' % (dump_build.http_log_url(), zip_name) build._log('test-migration', 'Restoring dump [%s](%s) from build [%s](%s)' % (zip_name, dump_url, dump_build.id, dump_build.build_url), log_type='markdown') - restore_suffix = self.restore_rename_db_suffix or params.dump_db.db_suffix + restore_suffix = self.restore_rename_db_suffix or params.dump_db.db_suffix or suffix assert restore_suffix restore_db_name = '%s-%s' % (build.dest, restore_suffix) diff --git a/runbot/models/bundle.py b/runbot/models/bundle.py index 693dd36d..f25cccc8 100644 --- a/runbot/models/bundle.py +++ b/runbot/models/bundle.py @@ -240,21 +240,3 @@ class Bundle(models.Model): for branch in self.branch_ids.sorted(key=lambda b: (b.is_pr)): branch_groups[branch.remote_id.repo_id].append(branch) return branch_groups - - -class BundleTriggerCustomisation(models.Model): - _name = 'runbot.bundle.trigger.custom' - _description = 'Custom trigger' - - trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', bundle_id.project_id)]") - bundle_id = fields.Many2one('runbot.bundle') - config_id = fields.Many2one('runbot.build.config') - extra_params = fields.Char("Custom parameters") - - _sql_constraints = [ - ( - "bundle_custom_trigger_unique", - "unique (bundle_id, trigger_id)", - "Only one custom trigger per trigger per bundle is allowed", - ) - ] diff --git a/runbot/models/custom_trigger.py b/runbot/models/custom_trigger.py new file mode 100644 index 00000000..6cca1c19 --- /dev/null +++ b/runbot/models/custom_trigger.py @@ -0,0 +1,97 @@ +import json + +from odoo import models, fields, api +from ..fields import JsonDictField + +class BundleTriggerCustomization(models.Model): + _name = 'runbot.bundle.trigger.custom' + _description = 'Custom trigger' + + trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', bundle_id.project_id)]") + bundle_id = fields.Many2one('runbot.bundle') + config_id = fields.Many2one('runbot.build.config') + extra_params = fields.Char("Custom parameters") + config_data = JsonDictField("Config data") + + _sql_constraints = [ + ( + "bundle_custom_trigger_unique", + "unique (bundle_id, trigger_id)", + "Only one custom trigger per trigger per bundle is allowed", + ) + ] + +class CustomTriggerWizard(models.TransientModel): + _name = 'runbot.trigger.custom.wizard' + _description = 'Custom trigger Wizard' + + bundle_id = fields.Many2one('runbot.bundle', "Bundle") + project_id = fields.Many2one(related='bundle_id.project_id', string='Project') + trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', project_id)]") + config_id = fields.Many2one('runbot.build.config', string="Config id", default=lambda self: self.env.ref('runbot.runbot_build_config_custom_multi')) + + config_data = fields.Text("Config data") # Text, hack to make it editable waiting for json widget + + number_build = fields.Integer('Number builds for config multi', default=10) + + child_extra_params = fields.Char('Extra params for children', default='--test-tags /module.test_method') + child_dump_url = fields.Char('Dump url for children') + child_config_id = fields.Many2one('runbot.build.config', 'Config for children', default=lambda self: self.env.ref('runbot.runbot_build_config_restore_and_test')) + + warnings = fields.Text('Warnings', readonly=True) + + @api.onchange('child_extra_params', 'child_dump_url', 'child_config_id', 'number_build', 'config_id', 'trigger_id') + def _onchange_warnings(self): + for wizard in self: + _warnings = [] + if wizard._get_existing_trigger(): + _warnings.append(f'A custom trigger already exists for trigger {wizard.trigger_id.name} and will be unlinked') + + if wizard.child_dump_url or wizard.child_extra_params or wizard.child_config_id or wizard.number_build: + if not any(step.job_type == 'create_build' for step in wizard.config_id.step_ids()): + _warnings.append('Some multi builds params are given but config as no create step') + + if wizard.child_dump_url and not any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()): + _warnings.append('A dump_url is defined but child config has no restore step') + + if not wizard.child_dump_url and any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()): + _warnings.append('Child config has a restore step but no dump_url is given') + + if not wizard.trigger_id.manual: + _warnings.append("This custom trigger will replace an existing non manual trigger. The ci won't be sent anymore") + + wizard.warnings = '\n'.join(_warnings) + + @api.onchange('number_build', 'child_extra_params', 'child_dump_url', 'child_config_id') + def _onchange_config_data(self): + for wizard in self: + config_data = self._get_config_data() + wizard.config_data = json.dumps(config_data, indent=True) + + def _get_config_data(self): + config_data = {} + if self.number_build: + config_data['number_build'] = self.number_build + child_data = {} + if self.child_extra_params: + child_data['extra_params'] = self.child_extra_params + if self.child_dump_url: + child_data['config_data'] = {'dump_url': self.child_dump_url} + if self.child_config_id: + child_data['config_id'] = self.child_config_id.id + if child_data: + config_data['child_data'] = child_data + return config_data + + def _get_existing_trigger(self): + return self.env['runbot.bundle.trigger.custom'].search([('bundle_id', '=', self.bundle_id.id), ('trigger_id', '=', self.trigger_id.id)]) + + def submit(self): + self.ensure_one() + self._get_existing_trigger().unlink() + self.env['runbot.bundle.trigger.custom'].create({ + 'bundle_id': self.bundle_id.id, + 'trigger_id': self.trigger_id.id, + 'config_id': self.config_id.id, + 'config_data': json.loads(self.config_data), + }) diff --git a/runbot/views/custom_trigger_wizard_views.xml b/runbot/views/custom_trigger_wizard_views.xml new file mode 100644 index 00000000..5d3b065f --- /dev/null +++ b/runbot/views/custom_trigger_wizard_views.xml @@ -0,0 +1,38 @@ + + + + runbot_trigger_custom_wizard + runbot.trigger.custom.wizard + +
+ + + + + + + + + + + + +
+
+
+
+
+ + + Generate custom trigger + runbot.trigger.custom.wizard + form + + new + + form + {'default_bundle_id': active_id} + +
diff --git a/runbot/wizards/__init__.py b/runbot/wizards/__init__.py index aa69f76c..e1c64da3 100644 --- a/runbot/wizards/__init__.py +++ b/runbot/wizards/__init__.py @@ -1,4 +1,3 @@ # -*- coding: utf-8 -*- -from . import multi_build_wizard from . import stat_regex_wizard diff --git a/runbot/wizards/multi_build_wizard.py b/runbot/wizards/multi_build_wizard.py deleted file mode 100644 index e8c52a7d..00000000 --- a/runbot/wizards/multi_build_wizard.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- - -from odoo import fields, models, api - - -class MultiBuildWizard(models.TransientModel): - - _name = 'runbot.build.config.multi.wizard' - _description = "Multi wizard" - - base_name = fields.Char('Generic name', required=True) - prefix = fields.Char('Prefix', help="Leave blank to use login.") - config_multi_name = fields.Char('Config name') - step_create_multi_name = fields.Char('Create multi step name') - config_single_name = fields.Char('Config only name') - config_single_extra_params = fields.Char('Extra cmd args') - config_single_test_tags = fields.Char('Test tags', default='') - config_single_test_enable = fields.Boolean('Enable tests', default=True) - step_single_name = fields.Char('Only step name') - number_builds = fields.Integer('Number of multi builds', default=10) - modules = fields.Char('Modules to install', help="List of module patterns to install, use * to install all available modules, prefix the pattern with dash to remove the module.", default='') - - @api.onchange('base_name', 'prefix') - def _onchange_name(self): - if self.base_name: - prefix = self.env.user.login.split('@')[0] if not self.prefix else self.prefix - self.prefix = prefix - name = '%s %s' % (prefix, self.base_name.capitalize()) - step_name = name.replace(' ', '_').lower() - - self.config_multi_name = '%s Multi' % name - self.step_create_multi_name = '%s_create_multi' % step_name - self.config_single_name = '%s Single' % name - self.step_single_name = '%s_single' % step_name - - def generate(self): - if self.base_name: - # Create the "only" step and config - step_single = self.env['runbot.build.config.step'].create({ - 'name': self.step_single_name, - 'job_type': 'install_odoo', - 'test_tags': self.config_single_test_tags, - 'extra_params': self.config_single_extra_params, - 'test_enable': self.config_single_test_enable, - 'install_modules': self.modules, - }) - config_single = self.env['runbot.build.config'].create({'name': self.config_single_name}) - - self.env['runbot.build.config.step.order'].create({ - 'sequence': 10, - 'config_id': config_single.id, - 'step_id': step_single.id - }) - - # Create the multiple builds step and config - step_create_multi = self.env['runbot.build.config.step'].create({ - 'name': self.step_create_multi_name, - 'job_type': 'create_build', - 'create_config_ids': [(4, config_single.id)], - 'number_builds': self.number_builds, - }) - - config_multi = self.env['runbot.build.config'].create({'name': self.config_multi_name}) - - config_multi.group = config_multi - step_create_multi.group = config_multi - config_single.group = config_multi - step_single.group = config_multi - - self.env['runbot.build.config.step.order'].create({ - 'sequence': 10, - 'config_id': config_multi.id, - 'step_id': step_create_multi.id - }) diff --git a/runbot/wizards/mutli_build_wizard_views.xml b/runbot/wizards/mutli_build_wizard_views.xml deleted file mode 100644 index 9741b962..00000000 --- a/runbot/wizards/mutli_build_wizard_views.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - runbot_multi_build_wizard - runbot.build.config.multi.wizard - -
- - - - - - - - - - - - - -
-
-
-
-
- - - Generate Multi Build Config - ir.actions.act_window - runbot.build.config.multi.wizard - form - - new - - - - -