NextERP/runbot
5
0
Fork 0
mirror of https://github.com/odoo/runbot.git synced 2025-03-27 13:25:47 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

[IMP] runbot: docker failure management

Browse Source
This commit is contained in:
Xavier-Do 2023-03-17 11:00:59 +01:00 committed by Christophe Monniez
parent 0e14e6d922
commit 44ec541e32
1 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
runbot/container.py
View File

@ -14,6 +14,7 @@ import logging
import os import os
import re import re
import subprocess import subprocess
import time
import warnings import warnings
# unsolved issue https://github.com/docker/docker-py/issues/2928 # unsolved issue https://github.com/docker/docker-py/issues/2928
@ -27,6 +28,7 @@ with warnings.catch_warnings():
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
docker_stop_failures = {}
class Command(): class Command():
@ -114,7 +116,6 @@ def _docker_build(build_dir, image_tag):
_logger.error('Build of image %s failed with this BUILD error:', image_tag) _logger.error('Build of image %s failed with this BUILD error:', image_tag)
msg = f"{e.msg}\n{''.join(l.get('stream') or '' for l in e.build_log)}" msg = f"{e.msg}\n{''.join(l.get('stream') or '' for l in e.build_log)}"
return (False, msg) return (False, msg)
_logger.info('Dockerfile %s finished build', image_tag)
return (True, None) return (True, None)
@ -203,6 +204,13 @@ def _docker_stop(container_name, build_dir):
"""Stops the container named container_name""" """Stops the container named container_name"""
container_name = sanitize_container_name(container_name) container_name = sanitize_container_name(container_name)
_logger.info('Stopping container %s', container_name) _logger.info('Stopping container %s', container_name)
if container_name in docker_stop_failures:
if docker_stop_failures[container_name] + 60 * 60 < time.time():
_logger.warning('Removing %s from docker_stop_failures', container_name)
del docker_stop_failures[container_name]
else:
_logger.warning('Skipping %s, is in failure', container_name)
return
docker_client = docker.from_env() docker_client = docker.from_env()
if build_dir: if build_dir:
end_file = os.path.join(build_dir, 'end-%s' % container_name) end_file = os.path.join(build_dir, 'end-%s' % container_name)
@ -212,10 +220,13 @@ def _docker_stop(container_name, build_dir):
try: try:
container = docker_client.containers.get(container_name) container = docker_client.containers.get(container_name)
container.stop(timeout=1) container.stop(timeout=1)
return
except docker.errors.NotFound: except docker.errors.NotFound:
_logger.error('Cannnot stop container %s. Container not found', container_name) _logger.error('Cannnot stop container %s. Container not found', container_name)
except docker.errors.APIError as e: except docker.errors.APIError as e:
_logger.error('Cannnot stop container %s. API Error "%s"', container_name, e) _logger.error('Cannnot stop container %s. API Error "%s"', container_name, e)
docker_stop_failures[container_name] = time.time()
def docker_state(container_name, build_dir): def docker_state(container_name, build_dir):
container_name = sanitize_container_name(container_name) container_name = sanitize_container_name(container_name)