[IMP] runbot: various build_config improvements

runbot/models/build_config.py

@@ -175,7 +175,7 @@ class ConfigStep(models.Model):
     upgrade_to_all_versions = fields.Boolean() # upgrade niglty (no master)
     upgrade_to_version_ids = fields.Many2many('runbot.version', relation='runbot_upgrade_to_version_ids', string='Forced version to use as target')
     # 2. define source from target
-    #upgrade_from_current = fields.Boolean()  #usefull for future migration (13.0-dev/13.3-dev  -> master) AVOID TO USE THAT
+    upgrade_from_current = fields.Boolean(help="If checked, only upgrade from current will be used, other options will be ignored Template should be installed in the same build")
     upgrade_from_previous_major_version = fields.Boolean() # 13.0
     upgrade_from_last_intermediate_version = fields.Boolean() # 13.3
     upgrade_from_all_intermediate_version = fields.Boolean() # 13.2 # 13.1
@@ -277,17 +277,21 @@ class ConfigStep(models.Model):
         count = 0
         config_data = build.params_id.config_data
         config_ids = config_data.get('create_config_ids', self.create_config_ids)
-        for create_config in config_ids:
-            child_data = {'config_id': create_config.id}
-            if 'child_data' in config_data:
-                child_data.update(config_data['child_data'])
-            for _ in range(config_data.get('number_build', self.number_builds)):
-                count += 1
-                if count > 200:
-                    build._logger('Too much build created')
-                    break
-                child = build._add_child(child_data, orphan=self.make_orphan)
-                build._log('create_build', 'created with config %s' % create_config.name, log_type='subbuild', path=str(child.id))
+
+        child_data_list = config_data.get('child_data',[{}])
+        if not isinstance(child_data_list, list):
+            child_data_list = [child_data_list]
+
+        for child_data in child_data_list:
+            for create_config in self.env['runbot.build.config'].browse(child_data.get('config_id', config_ids.ids)):
+                _child_data = {**child_data, 'config_id': create_config}
+                for _ in range(_child_data.get('number_build', self.number_builds)):
+                    count += 1
+                    if count > 200:
+                        build._logger('Too much build created')
+                        break
+                    child = build._add_child(_child_data, orphan=self.make_orphan)
+                    build._log('create_build', 'created with config %s' % create_config.name, log_type='subbuild', path=str(child.id))
 
 
     def make_python_ctx(self, build):
@@ -590,8 +594,11 @@ class ConfigStep(models.Model):
             if param.upgrade_from_build_id:
                 source_builds_by_target[target_build] = param.upgrade_from_build_id
             else:
-                target_version = target_build.params_id.version_id
-                from_builds = self._get_upgrade_source_builds(target_version, builds_references_by_version_id)
+                if self.upgrade_from_current:
+                    from_builds = build
+                else:
+                    target_version = target_build.params_id.version_id
+                    from_builds = self._get_upgrade_source_builds(target_version, builds_references_by_version_id)
                 source_builds_by_target[target_build] = from_builds
                 if from_builds:
                     build._log('', 'Defining source version(s) for %s: %s' % (target_build.params_id.version_id.name, ', '.join(source_builds_by_target[target_build].mapped('params_id.version_id.name'))))
@@ -606,10 +613,11 @@ class ConfigStep(models.Model):
             return  # replace this by a python job friendly solution
 
         assert not param.dump_db
-        if not self.upgrade_dbs:
-            build._log('configure_upgrade', 'No upgrade dbs defined in step %s' % self.name, level='WARN')
         for target, sources in source_builds_by_target.items():
             for source in sources:
+                valid_databases = []
+                if not self.upgrade_dbs:
+                    valid_databases = source.database_ids
                 for upgrade_db in self.upgrade_dbs:
                     if not upgrade_db.min_target_version_id or upgrade_db.min_target_version_id.number <= target.params_id.version_id.number:
                         config_id = upgrade_db.config_id
@@ -618,32 +626,32 @@ class ConfigStep(models.Model):
                         if not dump_builds:
                             build._log('_run_configure_upgrade', 'No child build found with config %s in %s' % (config_id.name, source.id), level='ERROR')
                         dbs = dump_builds.database_ids.sorted('db_suffix')
-                        valid_databases = list(self._filter_upgrade_database(dbs, upgrade_db.db_pattern))
+                        valid_databases += list(self._filter_upgrade_database(dbs, upgrade_db.db_pattern))
                         if not valid_databases:
                             build._log('_run_configure_upgrade', 'No datase found for pattern %s' % (upgrade_db.db_pattern), level='ERROR')
-                        for db in valid_databases:
-                            #commit_ids = build.params_id.commit_ids
-                            #if commit_ids != target.params_id.commit_ids:
-                            #    repo_ids = commit_ids.mapped('repo_id')
-                            #    for commit_link in target.params_id.commit_link_ids:
-                            #        if commit_link.commit_id.repo_id not in repo_ids:
-                            #            additionnal_commit_links |= commit_link
-                            #    build._log('', 'Adding sources from build [%s](%s)' % (target.id, target.build_url), log_type='markdown')
+                for db in valid_databases:
+                    #commit_ids = build.params_id.commit_ids
+                    #if commit_ids != target.params_id.commit_ids:
+                    #    repo_ids = commit_ids.mapped('repo_id')
+                    #    for commit_link in target.params_id.commit_link_ids:
+                    #        if commit_link.commit_id.repo_id not in repo_ids:
+                    #            additionnal_commit_links |= commit_link
+                    #    build._log('', 'Adding sources from build [%s](%s)' % (target.id, target.build_url), log_type='markdown')
 
-                            child = build._add_child({
-                                'upgrade_to_build_id': target.id,
-                                'upgrade_from_build_id': source,
-                                'dump_db': db.id,
-                                'config_id': self.upgrade_config_id
-                            })
+                    child = build._add_child({
+                        'upgrade_to_build_id': target.id,
+                        'upgrade_from_build_id': source,
+                        'dump_db': db.id,
+                        'config_id': self.upgrade_config_id
+                    })
 
-                            child.description = 'Testing migration from %s to %s using db %s (%s)' % (
-                                source.params_id.version_id.name,
-                                target.params_id.version_id.name,
-                                db.name,
-                                config_id.name
-                            )
-                        # TODO log somewhere if no db at all is found for a db_suffix
+                    child.description = 'Testing migration from %s to %s using db %s (%s)' % (
+                        source.params_id.version_id.name,
+                        target.params_id.version_id.name,
+                        db.name,
+                        config_id.name
+                    )
+                # TODO log somewhere if no db at all is found for a db_suffix
 
     def _get_upgrade_source_versions(self, target_version):
         if self.upgrade_from_version_ids:

runbot/tests/test_build_config_step.py

@@ -4,11 +4,9 @@ from odoo.exceptions import UserError
 from odoo.addons.runbot.common import RunbotException
 from .common import RunbotCase
 
-
-class TestBuildConfigStep(RunbotCase):
-
+class TestBuildConfigStepCommon(RunbotCase):
     def setUp(self):
-        super(TestBuildConfigStep, self).setUp()
+        super().setUp()
 
         self.Build = self.env['runbot.build']
         self.ConfigStep = self.env['runbot.build.config.step']
@@ -24,20 +22,24 @@ class TestBuildConfigStep(RunbotCase):
         self.start_patcher('find_patcher', 'odoo.addons.runbot.common.find', 0)
         self.start_patcher('findall_patcher', 'odoo.addons.runbot.models.build.BuildResult.parse_config', {})
 
-    def test_config_step_create_results(self):
-        """ Test child builds are taken into account"""
 
-        config_step = self.ConfigStep.create({
+class TestBuildConfigStepCreate(TestBuildConfigStepCommon):
+
+    def setUp(self):
+        super().setUp()
+        self.config_step = self.ConfigStep.create({
             'name': 'test_step',
             'job_type': 'create_build',
             'number_builds': 2,
-            'make_orphan': False,
         })
+        self.child_config = self.Config.create({'name': 'test_config'})
+        self.config_step.create_config_ids = [self.child_config.id]
 
-        config = self.Config.create({'name': 'test_config'})
-        config_step.create_config_ids = [config.id]
+    def test_config_step_create_results(self):
+        """ Test child builds are taken into account"""
 
-        config_step._run_create_build(self.parent_build, '/tmp/essai')
+
+        self.config_step._run_create_build(self.parent_build, '/tmp/essai')
         self.assertEqual(len(self.parent_build.children_ids), 2, 'Two sub-builds should have been generated')
 
         # check that the result will be ignored by parent build
@@ -50,18 +52,8 @@ class TestBuildConfigStep(RunbotCase):
 
     def test_config_step_create(self):
         """ Test the config step of type create """
-
-        config_step = self.ConfigStep.create({
-            'name': 'test_step',
-            'job_type': 'create_build',
-            'number_builds': 2,
-            'make_orphan': True,
-        })
-
-        config = self.Config.create({'name': 'test_config'})
-        config_step.create_config_ids = [config.id]
-
-        config_step._run_create_build(self.parent_build, '/tmp/essai')
+        self.config_step.make_orphan = True
+        self.config_step._run_create_build(self.parent_build, '/tmp/essai')
         self.assertEqual(len(self.parent_build.children_ids), 2, 'Two sub-builds should have been generated')
 
         # check that the result will be ignored by parent build
@@ -71,6 +63,73 @@ class TestBuildConfigStep(RunbotCase):
 
         self.assertFalse(self.parent_build.global_result)
 
+    def test_config_step_create_child_data(self):
+        """ Test the config step of type create """
+        self.config_step.number_builds = 5
+        json_config = {'child_data': [{'extra_params': '-i m1'}, {'extra_params': '-i m2'}]}
+        self.parent_build = self.Build.create({
+            'params_id': self.base_params.create({
+                'version_id': self.version_13.id,
+                'project_id': self.project.id,
+                'config_id': self.default_config.id,
+                'config_data': json_config,
+            }).id,
+        })
+
+        self.config_step._run_create_build(self.parent_build, '/tmp/essai')
+        self.assertEqual(len(self.parent_build.children_ids), 10, '10 build should have been generated')
+
+        # check that the result will be ignored by parent build
+        for child_build in self.parent_build.children_ids:
+            self.assertTrue(child_build.config_id, self.child_config)
+
+    def test_config_step_create_child_data_unique(self):
+        """ Test the config step of type create """
+        self.config_step.number_builds = 5
+        json_config = {'child_data': {'extra_params': '-i m1'}}
+        self.parent_build = self.Build.create({
+            'params_id': self.base_params.create({
+                'version_id': self.version_13.id,
+                'project_id': self.project.id,
+                'config_id': self.default_config.id,
+                'config_data': json_config,
+            }).id,
+        })
+
+        self.config_step._run_create_build(self.parent_build, '/tmp/essai')
+        self.assertEqual(len(self.parent_build.children_ids), 5, '5 build should have been generated')
+
+        # check that the result will be ignored by parent build
+        for child_build in self.parent_build.children_ids:
+            self.assertTrue(child_build.config_id, self.child_config)
+
+    def test_config_step_create_child_data_with_config(self):
+        """ Test the config step of type create """
+
+        test_config_1 = self.Config.create({'name': 'test_config1'})
+        test_config_2 = self.Config.create({'name': 'test_config2'})
+
+        self.config_step.number_builds = 5
+        json_config = {'child_data': [{'extra_params': '-i m1', 'config_id': test_config_1.id}, {'config_id': test_config_2.id}]}
+        self.parent_build = self.Build.create({
+            'params_id': self.base_params.create({
+                'version_id': self.version_13.id,
+                'project_id': self.project.id,
+                'config_id': self.default_config.id,
+                'config_data': json_config,
+            }).id,
+        })
+
+        self.config_step._run_create_build(self.parent_build, '/tmp/essai')
+        self.assertEqual(len(self.parent_build.children_ids), 10, '10 build should have been generated')
+        self.assertEqual(len(self.parent_build.children_ids.filtered(lambda b: b.config_id == test_config_1)), 5)
+        self.assertEqual(len(self.parent_build.children_ids.filtered(lambda b: b.config_id == test_config_2)), 5)
+
+
+
+
+class TestBuildConfigStep(TestBuildConfigStepCommon):
+
     def test_config_step_raises(self):
         """ Test a config raises when run step position is wrong"""