diff --git a/runbot/__manifest__.py b/runbot/__manifest__.py index 5085df56..84d15e5d 100644 --- a/runbot/__manifest__.py +++ b/runbot/__manifest__.py @@ -6,7 +6,7 @@ 'author': "Odoo SA", 'website': "http://runbot.odoo.com", 'category': 'Website', - 'version': '5.9', + 'version': '5.10', 'application': True, 'depends': ['base', 'base_automation', 'website'], 'data': [ diff --git a/runbot/container.py b/runbot/container.py index 4134c9d3..faabc2e0 100644 --- a/runbot/container.py +++ b/runbot/container.py @@ -137,6 +137,24 @@ def _docker_build(build_dir, image_tag): return dm.result +def docker_tag(identifier_or_tag, new_tag): + return _docker_tag(identifier_or_tag, new_tag) + +def _docker_tag(identifier_or_tag, new_tag): + if not identifier_or_tag: + return + _logger.info('Tagging image %s to "%s"', identifier_or_tag, new_tag) + docker_client = docker.from_env() + repo, tag = new_tag.split(':') # runbot DockerFile tags contains the repo part + try: + image = docker_client.images.get(identifier_or_tag) + image.tag(repo, tag) + except docker.errors.ImageNotFound: + _logger.warning('failed to find docker image with identifier %s', identifier_or_tag) + except docker.errors.APIError: + _logger.warning('failed to retag docker image with identifier %s', identifier_or_tag) + + def docker_push(image_tag, push_url='127.0.0.1:5001'): return _docker_push(image_tag, push_url) @@ -391,3 +409,12 @@ def sanitize_container_name(name): """Returns a container name with unallowed characters removed""" name = re.sub('^[^a-zA-Z0-9]+', '', name) return re.sub('[^a-zA-Z0-9_.-]', '', name) + + +def docker_get_identifier(tag_or_id): + docker_client = docker.from_env() + try: + image = docker_client.images.get(tag_or_id) + return image.short_id + except docker.errors.ImageNotFound: + return False diff --git a/runbot/migrations/18.0.5.10/post-migration.py b/runbot/migrations/18.0.5.10/post-migration.py new file mode 100644 index 00000000..f91d5ee9 --- /dev/null +++ b/runbot/migrations/18.0.5.10/post-migration.py @@ -0,0 +1,11 @@ +import logging + +_logger = logging.getLogger(__name__) + +def migrate(cr, version): + cr.execute(""" + UPDATE runbot_dockerfile + SET image_identifier = subq.identifier, image_future_identifier = subq.identifier + FROM (SELECT dockerfile_id, identifier FROM runbot_docker_build_result WHERE result='success' order by create_date desc) AS subq + WHERE runbot_dockerfile.id = subq.dockerfile_id; + """) diff --git a/runbot/models/docker.py b/runbot/models/docker.py index 6c407b2a..6221a833 100644 --- a/runbot/models/docker.py +++ b/runbot/models/docker.py @@ -124,7 +124,10 @@ class Dockerfile(models.Model): name = fields.Char('Dockerfile name', required=True, help="Name of Dockerfile") active = fields.Boolean('Active', default=True, tracking=True) + image_identifier = fields.Char('Identifier') + image_future_identifier = fields.Char('Future Identifier') image_tag = fields.Char(compute='_compute_image_tag', store=True) + image_future_tag = fields.Char(compute='_compute_image_future_tag') template_id = fields.Many2one('ir.ui.view', string='Docker Template', domain=[('type', '=', 'qweb')], context={'default_type': 'qweb', 'default_arch_base': ''}) arch_base = fields.Text(related='template_id.arch_base', readonly=False, related_sudo=True) dockerfile = fields.Text(compute='_compute_dockerfile', tracking=True) @@ -198,6 +201,11 @@ class Dockerfile(models.Model): if rec.name: rec.image_tag = 'odoo:%s' % re.sub(r'[ /:\(\)\[\]]', '', rec.name) + @api.depends('image_tag') + def _compute_image_future_tag(self): + for rec in self: + rec.image_future_tag = f'{rec.image_tag}.future' + @api.depends('template_id') def _compute_view_ids(self): for rec in self: @@ -332,7 +340,7 @@ class Dockerfile(models.Model): with open(self.env['runbot.runbot']._path('docker', self.image_tag, 'Dockerfile'), 'w') as Dockerfile: Dockerfile.write(content) - result = docker_build(docker_build_path, self.image_tag) + result = docker_build(docker_build_path, self.image_future_tag) duration = result['duration'] msg = result['msg'] success = image_id = result.get('image_id') @@ -372,6 +380,7 @@ class Dockerfile(models.Model): message = f'Build failure, check results for more info ({result.summary})' self.message_post(body=message) _logger.error(message) + return image_id class DockerBuildOutput(models.Model): diff --git a/runbot/models/host.py b/runbot/models/host.py index 8db11c3a..6f423ad9 100644 --- a/runbot/models/host.py +++ b/runbot/models/host.py @@ -6,7 +6,7 @@ from docker.errors import ImageNotFound from odoo import models, fields, api from odoo.tools import config, ormcache from ..common import fqdn, local_pgadmin_cursor, os, list_local_dbs, local_pg_cursor -from ..container import docker_push, docker_pull, docker_prune, docker_images, docker_remove +from ..container import docker_push, docker_pull, docker_prune, docker_images, docker_remove, docker_tag _logger = logging.getLogger(__name__) @@ -150,7 +150,7 @@ class Host(models.Model): # pull all images from the runbot docker registry is_registry = docker_registry_host == self all_docker_files = self.env['runbot.dockerfile'].search([]) - all_tags = set(all_docker_files.mapped('image_tag')) + all_tags = set(all_docker_files.mapped('image_tag')) | set(all_docker_files.mapped('image_future_tag')) if docker_registry_url and self.use_remote_docker_registry and not is_registry: _logger.info('Pulling docker images...') total_duration = 0 @@ -166,18 +166,22 @@ class Host(models.Model): else: _logger.info('Building docker images...') for dockerfile in self.env['runbot.dockerfile'].search([('to_build', '=', True)]): - dockerfile._build(self) + identifier = dockerfile._build(self) + dockerfile.image_future_identifier = identifier + docker_tag(dockerfile.image_identifier, dockerfile.image_tag) + docker_tag(dockerfile.image_future_identifier, dockerfile.image_future_tag) if is_registry: - try: - docker_push(dockerfile.image_tag) # for now, always push locally - if self.docker_registry_url: - docker_registry_url = self.docker_registry_url - else: - docker_registry_url = icp.get_param('runbot.docker_registry_url', default='').strip('/') - if docker_registry_url: - docker_push(dockerfile.image_tag, docker_registry_url) - except ImageNotFound: - _logger.warning("Image tag `%s` not found. Skipping push", dockerfile.image_tag) + for tag in [dockerfile.image_tag, dockerfile.image_future_tag]: + try: + docker_push(tag) # for now, always push locally + if self.docker_registry_url: + docker_registry_url = self.docker_registry_url + else: + docker_registry_url = icp.get_param('runbot.docker_registry_url', default='').strip('/') + if docker_registry_url: + docker_push(tag, docker_registry_url) + except ImageNotFound: + _logger.warning("Image tag `%s` not found. Skipping push", dockerfile.image_future_tag) _logger.info('Cleaning docker images...') for image in docker_images(): diff --git a/runbot/models/res_config_settings.py b/runbot/models/res_config_settings.py index 90a60aab..9d29f005 100644 --- a/runbot/models/res_config_settings.py +++ b/runbot/models/res_config_settings.py @@ -51,7 +51,8 @@ class ResConfigSettings(models.TransientModel): runbot_pending_warning = fields.Integer('Pending warning limit', default=5, config_parameter='runbot.pending.warning') runbot_pending_critical = fields.Integer('Pending critical limit', default=5, config_parameter='runbot.pending.critical') - runbot_docker_registry_host_id = fields.Many2one('runbot.host', 'Docker registry', help='Runbot host which handles Docker registry.', config_parameter='runbot.docker_registry_host_id') + runbot_docker_registry_host_id = fields.Many2one('runbot.host', 'Docker builder', help='Runbot host which handles Docker builds.', config_parameter='runbot.docker_registry_host_id') + runbot_docker_registry_url = fields.Char('Docker Registry url', help='Remote Registry Url', config_parameter='runbot.docker_registry_url') # TODO other icp # runbot.runbot_maxlogs 100 # migration db diff --git a/runbot/tests/common.py b/runbot/tests/common.py index abc7ed4f..cc981a76 100644 --- a/runbot/tests/common.py +++ b/runbot/tests/common.py @@ -181,6 +181,11 @@ class RunbotCase(TransactionCase): self.start_patcher('docker_run', 'odoo.addons.runbot.container._docker_run') self.start_patcher('docker_build', 'odoo.addons.runbot.container._docker_build') self.start_patcher('docker_push', 'odoo.addons.runbot.container._docker_push') + self.start_patcher('docker_prune', 'odoo.addons.runbot.container._docker_prune') + self.start_patcher('docker_pull', 'odoo.addons.runbot.container._docker_pull') + self.start_patcher('docker_tag', 'odoo.addons.runbot.container._docker_tag') + self.start_patcher('docker_images', 'odoo.addons.runbot.container._docker_images') + self.start_patcher('docker_remove', 'odoo.addons.runbot.container._docker_remove') self.start_patcher('docker_ps', 'odoo.addons.runbot.container._docker_ps', []) self.start_patcher('docker_stop', 'odoo.addons.runbot.container._docker_stop') self.start_patcher('docker_get_gateway_ip', 'odoo.addons.runbot.models.build_config.docker_get_gateway_ip', None) @@ -196,7 +201,6 @@ class RunbotCase(TransactionCase): self.start_patcher('getmtime', 'odoo.addons.runbot.common.os.path.getmtime', datetime.datetime.now().timestamp()) self.start_patcher('file_exist', 'odoo.tools.misc.os.path.exists', True) - self.start_patcher('_get_py_version', 'odoo.addons.runbot.models.build.BuildResult._get_py_version', 3) def no_commit(*_args, **_kwargs): diff --git a/runbot/tests/test_host.py b/runbot/tests/test_host.py index 2aeb24f5..6f119da5 100644 --- a/runbot/tests/test_host.py +++ b/runbot/tests/test_host.py @@ -1,5 +1,7 @@ import logging +from unittest.mock import call + from .common import RunbotCase from datetime import datetime, timedelta @@ -111,3 +113,45 @@ class TestHost(RunbotCase): self.patchers['fetch_local_logs'].return_value = logs self.test_host._process_logs() self.patchers['host_local_pg_cursor'].assert_called() + + def test_docker_builder(self): + self.start_patcher('build_patcher', 'odoo.addons.runbot.models.docker.Dockerfile._build') + + # deactivate DockerDefault to avoid test pollution + self.env.ref('runbot.docker_default').active = False + + icp = self.env['ir.config_parameter'] + icp.set_param('runbot.docker_registry_host_id', self.test_host.id) + icp.set_param('runbot.docker_registry_url', 'registryhost_nowhere') + dockerfile = self.env['runbot.dockerfile'].create({ + 'name': 'Docker Test', + 'to_build': True, + 'image_identifier': 'current', + 'image_future_identifier': 'current' + }) + + self.assertEqual(dockerfile.image_tag, 'odoo:DockerTest') + self.assertEqual(dockerfile.image_future_tag, 'odoo:DockerTest.future') + self.patchers['build_patcher'].side_effect = lambda x: 'future' + self.test_host._docker_update_images() + + self.assertEqual(dockerfile.image_future_identifier, 'future') + + expected_docker_tag_calls = [ + call('current', 'odoo:DockerTest'), + call('future', 'odoo:DockerTest.future') + ] + + self.patchers['docker_tag'].assert_has_calls(expected_docker_tag_calls) + + expected_push_calls = [ + call('odoo:DockerTest', '127.0.0.1:5001'), + call('odoo:DockerTest', 'registryhost_nowhere'), + call('odoo:DockerTest.future', '127.0.0.1:5001'), + call('odoo:DockerTest.future', 'registryhost_nowhere') + ] + + self.patchers['docker_push'].assert_has_calls(expected_push_calls) + self.patchers['docker_pull'].assert_not_called() + + diff --git a/runbot/views/dockerfile_views.xml b/runbot/views/dockerfile_views.xml index 62de7ce9..a75eeaf1 100644 --- a/runbot/views/dockerfile_views.xml +++ b/runbot/views/dockerfile_views.xml @@ -11,6 +11,8 @@ + + diff --git a/runbot/views/res_config_settings_views.xml b/runbot/views/res_config_settings_views.xml index 745fdf5f..f6d864b7 100644 --- a/runbot/views/res_config_settings_views.xml +++ b/runbot/views/res_config_settings_views.xml @@ -97,6 +97,9 @@ + + +