diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index 5f70e5f5..3f8564be 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -741,17 +741,36 @@ class ConfigStep(models.Model): if 'dump_url' in params.config_data: dump_url = params.config_data['dump_url'] zip_name = dump_url.split('/')[-1] - build._log('test-migration', 'Restoring db [%s](%s)' % (zip_name, dump_url), log_type='markdown') + build._log('_run_restore', f'Restoring db [{zip_name}]({dump_url})', log_type='markdown') suffix = 'all' else: - download_db_suffix = params.dump_db.db_suffix or self.restore_download_db_suffix - dump_build = params.dump_db.build_id or build.parent_id + if 'dump_trigger_id' in params.config_data: + dump_trigger = self.env['runbot.trigger'].browse(params.config_data['dump_trigger_id']) + dump_suffix = params.config_data.get('dump_suffix', 'all') + base_batch = build.params_id.create_batch_id.base_reference_batch_id + reference_build = base_batch.slot_ids.filtered(lambda s: s.trigger_id == dump_trigger).mapped('build_id') + if not reference_build: + build._log('_run_restore', f'No reference build found in batch {base_batch.id} for trigger {dump_trigger.name}', log_type='markdown', level='ERROR') + build._kill(result='ko') + return + if reference_build.local_state not in ('done', 'running'): + build._log('_run_restore', f'Reference build [{reference_build.id}]({reference_build.build_url} is not yet finished, database may not exist', log_type='markdown', level='WARNING') + dump_db = reference_build.database_ids.filtered(lambda d: d.db_suffix == dump_suffix) + if not dump_db: + build._log('_run_restore', f'No dump with suffix {dump_suffix} found in build [{reference_build.id}]({reference_build.build_url})', log_type='markdown', level='ERROR') + build._kill(result='ko') + return + else: + dump_db = params.dump_db + + download_db_suffix = dump_db.db_suffix or self.restore_download_db_suffix + dump_build = dump_db.build_id or build.parent_id assert download_db_suffix and dump_build download_db_name = '%s-%s' % (dump_build.dest, download_db_suffix) zip_name = '%s.zip' % download_db_name dump_url = '%s%s' % (dump_build.http_log_url(), zip_name) build._log('test-migration', 'Restoring dump [%s](%s) from build [%s](%s)' % (zip_name, dump_url, dump_build.id, dump_build.build_url), log_type='markdown') - restore_suffix = self.restore_rename_db_suffix or params.dump_db.db_suffix or suffix + restore_suffix = self.restore_rename_db_suffix or dump_db.db_suffix or suffix assert restore_suffix restore_db_name = '%s-%s' % (build.dest, restore_suffix) diff --git a/runbot/models/bundle.py b/runbot/models/bundle.py index 3355e5b8..7710eef7 100644 --- a/runbot/models/bundle.py +++ b/runbot/models/bundle.py @@ -248,3 +248,35 @@ class Bundle(models.Model): for branch in self.branch_ids.sorted(key=lambda b: (b.is_pr)): branch_groups[branch.remote_id.repo_id].append(branch) return branch_groups + + def generate_custom_trigger_multi_action(self): + context = { + 'default_bundle_id': self.id, + 'default_config_id': self.env.ref('runbot.runbot_build_config_custom_multi').id, + 'default_child_config_id': self.env.ref('runbot.runbot_build_config_restore_and_test').id, + 'default_extra_params': False, + 'default_child_extra_params': '--test-tags /module.test_method', + 'default_number_build': 10, + } + return self._generate_custom_trigger_action(context) + + def generate_custom_trigger_restore_action(self): + context = { + 'default_bundle_id': self.id, + 'default_config_id': self.env.ref('runbot.runbot_build_config_restore_and_test').id, + 'default_child_config_id': False, + 'default_extra_params': '--test-tags /module.test_method', + 'default_child_extra_params': False, + 'default_number_build': 0, + } + return self._generate_custom_trigger_action(context) + + def _generate_custom_trigger_action(self, context): + return { + 'type': 'ir.actions.act_window', + 'name': 'Generate custom trigger', + 'view_mode': 'form', + 'res_model': 'runbot.trigger.custom.wizard', + 'target': 'new', + 'context': context, + } diff --git a/runbot/models/custom_trigger.py b/runbot/models/custom_trigger.py index cdb4955a..0e79be60 100644 --- a/runbot/models/custom_trigger.py +++ b/runbot/models/custom_trigger.py @@ -26,57 +26,135 @@ class CustomTriggerWizard(models.TransientModel): _name = 'runbot.trigger.custom.wizard' _description = 'Custom trigger Wizard' + # minimal config options bundle_id = fields.Many2one('runbot.bundle', "Bundle") project_id = fields.Many2one(related='bundle_id.project_id', string='Project') trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', project_id)]") config_id = fields.Many2one('runbot.build.config', string="Config id", default=lambda self: self.env.ref('runbot.runbot_build_config_custom_multi')) + # base options config_data = JsonDictField("Config data") + extra_params = fields.Char('Extra params', default='') + # restore options + restore_mode = fields.Selection([('auto', 'Auto'), ('url', 'Dump url')]) + restore_dump_url = fields.Char('Dump url for children') + restore_trigger_id = fields.Many2one('runbot.trigger', 'Trigger to restore a dump', domain="[('project_id', '=', project_id), ('manual', '=', False)]") + restore_database_suffix = fields.Char('Database suffix to restore', default='all') + + # create multi options number_build = fields.Integer('Number builds for config multi', default=10) - child_extra_params = fields.Char('Extra params for children', default='--test-tags /module.test_method') - child_dump_url = fields.Char('Dump url for children') child_config_id = fields.Many2one('runbot.build.config', 'Config for children', default=lambda self: self.env.ref('runbot.runbot_build_config_restore_and_test')) warnings = fields.Text('Warnings', readonly=True) - @api.onchange('child_extra_params', 'child_dump_url', 'child_config_id', 'number_build', 'config_id', 'trigger_id') + has_create_step = fields.Boolean("Hase create step", compute="_compute_has_create_step") + has_restore_step = fields.Boolean("Hase restore step", compute="_compute_has_restore_step") + has_child_with_restore_step = fields.Boolean("Child config has create step", compute="_compute_has_child_with_restore_step") + + @api.depends('config_id') + def _compute_has_create_step(self): + for record in self: + record.has_create_step = any(step.job_type == 'create_build' for step in self.config_id.step_ids()) + + @api.depends('config_id') + def _compute_has_restore_step(self): + for record in self: + record.has_restore_step = any(step.job_type == 'restore' for step in self.config_id.step_ids()) + + @api.depends('child_config_id') + def _compute_has_child_with_restore_step(self): + for record in self: + record.has_child_with_restore_step = record.child_config_id and any(step.job_type == 'restore' for step in self.child_config_id.step_ids()) + + @api.onchange('child_extra_params', 'restore_dump_url', 'config_id', 'child_config_id', 'number_build', 'config_id', 'restore_mode', 'restore_database_suffix', 'restore_trigger_id') def _onchange_warnings(self): for wizard in self: _warnings = [] + + if not wizard.trigger_id: + _warnings.append(f'No trigger id given (required and may automatically fix other issues)') + if wizard._get_existing_trigger(): _warnings.append(f'A custom trigger already exists for trigger {wizard.trigger_id.name} and will be unlinked') - if wizard.child_dump_url or wizard.child_extra_params or wizard.child_config_id or wizard.number_build: - if not any(step.job_type == 'create_build' for step in wizard.config_id.step_ids()): - _warnings.append('Some multi builds params are given but config as no create step') + if wizard.restore_mode: + if (not wizard.has_restore_step and not wizard.has_child_with_restore_step): + _warnings.append('A restore mode is defined but no config has a restore step') + elif not wizard.restore_mode: + if wizard.has_restore_step : + _warnings.append('Config has a restore step but no restore mode is given') + if wizard.has_child_with_restore_step: + _warnings.append('Child config has a restore step but no restore mode is given') + elif wizard.restore_mode == "url": + if not wizard.restore_dump_url: + _warnings.append('The restore mode is url but no dump_url is given') + elif wizard.restore_mode == "auto": + if not wizard.restore_trigger_id: + _warnings.append('The restore mode is auto but no restore trigger is given') + if not wizard.restore_database_suffix: + _warnings.append('The restore mode is auto but no db suffix is given') - if wizard.child_dump_url and not any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()): - _warnings.append('A dump_url is defined but child config has no restore step') - - if not wizard.child_dump_url and any(step.job_type == 'restore' for step in wizard.child_config_id.step_ids()): - _warnings.append('Child config has a restore step but no dump_url is given') + if wizard.has_create_step: + if not wizard.child_config_id: + _warnings.append('Config has a create step nut no child config given') + if not wizard.child_extra_params: + _warnings.append('Config has a create step nut no child extra param given') + if wizard.extra_params: + _warnings.append('You may change `Extra params` to `Extra params for children`') + else: + if wizard.child_extra_params: + _warnings.append('Extra params for children given but config has no create step') + if wizard.child_config_id: + _warnings.append('Config for children given but config has no create step') + if not wizard.extra_params: + _warnings.append('No extra params are given') if not wizard.trigger_id.manual: _warnings.append("This custom trigger will replace an existing non manual trigger. The ci won't be sent anymore") wizard.warnings = '\n'.join(_warnings) - @api.onchange('number_build', 'child_extra_params', 'child_dump_url', 'child_config_id') - def _onchange_config_data(self): + @api.onchange('trigger_id') + def _onchange_trigger_id(self): for wizard in self: - wizard.config_data = self._get_config_data() + if wizard.trigger_id: + wizard.restore_trigger_id = wizard.trigger_id.restore_trigger_id + if wizard.restore_trigger_id and not wizard.restore_mode: + wizard.restore_mode = 'auto' + self._onchange_config_data() + self._onchange_warnings() + + @api.onchange('number_build', 'child_extra_params', 'restore_dump_url', 'child_config_id', 'restore_trigger_id', 'restore_database_suffix', 'restore_mode') + def _onchange_config_data(self): + for wizard in self: + wizard.config_data = self._get_config_data() def _get_config_data(self): config_data = {} if self.number_build: config_data['number_build'] = self.number_build + if self.extra_params: + config_data['extra_params'] = self.extra_params child_data = {} if self.child_extra_params: child_data['extra_params'] = self.child_extra_params - if self.child_dump_url: - child_data['config_data'] = {'dump_url': self.child_dump_url} + if self.restore_mode: + restore_params = {} + if self.restore_mode == 'url': + if self.restore_dump_url: + restore_params['dump_url'] = self.restore_dump_url + else: + if self.restore_trigger_id: + restore_params['dump_trigger_id'] = self.restore_trigger_id.id + if self.restore_database_suffix: + restore_params['dump_suffix'] = self.restore_database_suffix + if self.has_child_with_restore_step: + child_data['config_data'] = restore_params + if not self.has_child_with_restore_step or self.has_restore_step: + config_data.update(restore_params) + if self.child_config_id: child_data['config_id'] = self.child_config_id.id if child_data: diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 973d74f0..390fea59 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -38,22 +38,23 @@ class Trigger(models.Model): sequence = fields.Integer('Sequence') name = fields.Char("Name") description = fields.Char("Description", help="Informative description") - project_id = fields.Many2one('runbot.project', string="Project id", required=True, default=lambda self: self.env.ref('runbot.main_project', raise_if_not_found=False)) + project_id = fields.Many2one('runbot.project', string="Project id", required=True) repo_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_triggers', string="Triggers", domain="[('project_id', '=', project_id)]") dependency_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_dependencies', string="Dependencies") config_id = fields.Many2one('runbot.build.config', string="Config", required=True) batch_dependent = fields.Boolean('Batch Dependent', help="Force adding batch in build parameters to make it unique and give access to bundle") - ci_context = fields.Char("Ci context", default='ci/runbot', tracking=True) + ci_context = fields.Char("CI context", tracking=True) category_id = fields.Many2one('runbot.category', default=lambda self: self.env.ref('runbot.default_category', raise_if_not_found=False)) version_domain = fields.Char(string="Version domain") hide = fields.Boolean('Hide trigger on main page') manual = fields.Boolean('Only start trigger manually', default=False) + restore_trigger_id = fields.Many2one('runbot.trigger', string='Restore Trigger ID for custom triggers', help="Mainly usefull to automatically define where to find a reference database when creating a custom trigger", tracking=True) upgrade_dumps_trigger_id = fields.Many2one('runbot.trigger', string='Template/complement trigger', tracking=True) upgrade_step_id = fields.Many2one('runbot.build.config.step', compute="_compute_upgrade_step_id", store=True) - ci_url = fields.Char("ci url") - ci_description = fields.Char("ci description") + ci_url = fields.Char("CI url") + ci_description = fields.Char("CI description") has_stats = fields.Boolean('Has a make_stats config step', compute="_compute_has_stats", store=True) team_ids = fields.Many2many('runbot.team', string="Runbot Teams", help="Teams responsible of this trigger, mainly usefull for nightly") @@ -276,8 +277,7 @@ class Repo(models.Model): main_remote_id = fields.Many2one('runbot.remote', "Main remote", tracking=True) remote_ids = fields.One2many('runbot.remote', 'repo_id', "Remotes") project_id = fields.Many2one('runbot.project', required=True, tracking=True, - help="Default bundle project to use when pushing on this repos", - default=lambda self: self.env.ref('runbot.main_project', raise_if_not_found=False)) + help="Default bundle project to use when pushing on this repos") # -> not verry usefull, remove it? (iterate on projects or contraints triggers: # all trigger where a repo is used must be in the same project. modules = fields.Char("Modules to install", help="Comma-separated list of modules to install and test.", tracking=True) diff --git a/runbot/tests/common.py b/runbot/tests/common.py index fd89543d..40d67137 100644 --- a/runbot/tests/common.py +++ b/runbot/tests/common.py @@ -114,7 +114,7 @@ class RunbotCase(TransactionCase): 'is_pr': False, 'head': self.initial_server_commit.id, }) - self.branch_server.bundle_id # compute + self.master_bundle = self.branch_server.bundle_id # compute self.dev_bundle = self.Bundle.create({ 'name': 'master-dev-tri', 'project_id': self.project.id diff --git a/runbot/tests/test_build_config_step.py b/runbot/tests/test_build_config_step.py index 09fe6da2..18bbba50 100644 --- a/runbot/tests/test_build_config_step.py +++ b/runbot/tests/test_build_config_step.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from unittest.mock import patch, mock_open from odoo import Command +from odoo.tools import mute_logger from odoo.exceptions import UserError from odoo.addons.runbot.common import RunbotException from .common import RunbotCase @@ -198,6 +199,65 @@ class TestCodeowner(TestBuildConfigStepCommon): 'Requesting review for pull request [base/server:1234](https://example.com/base/server/pull/1234): codeowner-team, team_01, team_02, team_js, team_py' ]) +class TestBuildConfigStepRestore(TestBuildConfigStepCommon): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.restore_config_step = cls.env['runbot.build.config.step'].create({ + 'name': 'restore', + 'job_type': 'restore', + }) + cls.restore_config = cls.env['runbot.build.config'].create({ + 'name': 'Restore', + 'step_order_ids': [ + (0, 0, {'sequence': 10, 'step_id': cls.restore_config_step.id}), + ], + }) + + def test_restore(self): + # setup master branch + master_batch = self.master_bundle._force() + with mute_logger('odoo.addons.runbot.models.batch'): + master_batch._prepare() + reference_slot = master_batch.slot_ids + trigger = reference_slot.trigger_id + self.assertEqual(trigger.name, 'Server trigger', 'Just checking that we have a single slot') + reference_build = reference_slot.build_id + self.env['runbot.database'].create({ + 'build_id': reference_build.id, + 'name': f'{reference_build.dest}-suffix', + }) + reference_build.local_state = 'done' + reference_build.local_result = 'ok' + + # custom trigger + config_data = { + 'dump_trigger_id': trigger.id, + 'dump_suffix': 'suffix', + } + self.env['runbot.bundle.trigger.custom'].create({ + 'bundle_id': self.dev_bundle.id, + 'config_id': self.restore_config.id, + 'trigger_id': trigger.id, + 'config_data': config_data, + }) + + # create dev build + dev_batch = self.dev_bundle._force() + with mute_logger('odoo.addons.runbot.models.batch'): + dev_batch._prepare() + dev_batch.base_reference_batch_id = master_batch # not tested, this is not the purpose of this test + dev_build = dev_batch.slot_ids.build_id + self.assertEqual(dev_build.params_id.config_data, config_data) + + docker_params = self.restore_config_step._run_restore(dev_build, '/tmp/logs') + cmds = docker_params['cmd'].split(' && ') + self.assertEqual(f'wget https://False/runbot/static/build/{reference_build.dest}/logs/{reference_build.dest}-suffix.zip', cmds[2]) + self.assertEqual(f'psql -q {dev_build.dest}-suffix < dump.sql', cmds[8]) + self.called=True + + class TestBuildConfigStepCreate(TestBuildConfigStepCommon): diff --git a/runbot/views/bundle_views.xml b/runbot/views/bundle_views.xml index dd60ca96..acc92635 100644 --- a/runbot/views/bundle_views.xml +++ b/runbot/views/bundle_views.xml @@ -36,6 +36,10 @@ runbot.bundle
+
+
diff --git a/runbot/views/custom_trigger_wizard_views.xml b/runbot/views/custom_trigger_wizard_views.xml index 5d3b065f..4189b125 100644 --- a/runbot/views/custom_trigger_wizard_views.xml +++ b/runbot/views/custom_trigger_wizard_views.xml @@ -5,17 +5,29 @@ runbot.trigger.custom.wizard + + - - - - - - - - - - + + + + + + + + + + + + + + + + + + + +