From 847622552f9f0a87f377ff3a7b838d499bf18d99 Mon Sep 17 00:00:00 2001 From: Christophe Monniez Date: Wed, 30 Jun 2021 15:45:41 +0200 Subject: [PATCH] [IMP] runbot: limit memory usage of containers In some conditions, it appears that a containerized build can eat up all memory of the container host. This leads to disturbance of other builds as the kernel OOM killer enters the dance. With this commit, the docker ability to limit memory usage of a container is used. The OOM killer will choose its victim among the container processes. The containers memory limit has to be set in the runbot settings. If not set, no memory limit is used. --- runbot/container.py | 11 ++++++----- runbot/models/build.py | 3 +++ runbot/models/res_config_settings.py | 12 ++++++++++++ runbot/views/res_config_settings_views.xml | 3 +++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/runbot/container.py b/runbot/container.py index 9934478e..bd5b87a3 100644 --- a/runbot/container.py +++ b/runbot/container.py @@ -8,17 +8,13 @@ When testing this file: the first parameter should be a directory containing Odoo. The second parameter is the exposed port """ -import argparse import configparser -import datetime import io import json import logging import os import re -import shutil import subprocess -import time _logger = logging.getLogger(__name__) @@ -121,7 +117,7 @@ def docker_run(*args, **kwargs): return _docker_run(*args, **kwargs) -def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False, image_tag=False, exposed_ports=None, cpu_limit=None, preexec_fn=None, ro_volumes=None, env_variables=None): +def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False, image_tag=False, exposed_ports=None, cpu_limit=None, memory=None, preexec_fn=None, ro_volumes=None, env_variables=None): """Run tests in a docker container :param run_cmd: command string to run in container :param log_path: path to the logfile that will contain odoo stdout and stderr @@ -130,6 +126,7 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False :param container_name: used to give a name to the container for later reference :param image_tag: Docker image tag name to select which docker image to use :param exposed_ports: if not None, starting at 8069, ports will be exposed as exposed_ports numbers + :param memory: memory limit in bytes for the container :params ro_volumes: dict of dest:source volumes to mount readonly in builddir :params env_variables: list of environment variables """ @@ -157,6 +154,10 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False '--shm-size=128m', '--init', ] + + if memory: + docker_command.append('--memory=%s' % memory) + if ro_volumes: for dest, source in ro_volumes.items(): logs.write("Adding readonly volume '%s' pointing to %s \n" % (dest, source)) diff --git a/runbot/models/build.py b/runbot/models/build.py index f5043045..c68bdb52 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -732,6 +732,9 @@ class BuildResult(models.Model): kwargs.update({'image_tag': self.params_id.dockerfile_id.image_tag}) if kwargs['image_tag'] != 'odoo:DockerDefault': self._log('Preparing', 'Using Dockerfile Tag %s' % kwargs['image_tag']) + containers_memory_limit = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_containers_memory', 0) + if containers_memory_limit and 'memory' not in kwargs: + kwargs['memory'] = containers_memory_limit * 1024 ** 3 docker_run(**kwargs) def _path(self, *l, **kw): diff --git a/runbot/models/res_config_settings.py b/runbot/models/res_config_settings.py index 35c0342c..03121718 100644 --- a/runbot/models/res_config_settings.py +++ b/runbot/models/res_config_settings.py @@ -10,6 +10,8 @@ class ResConfigSettings(models.TransientModel): _inherit = 'res.config.settings' runbot_workers = fields.Integer('Default number of workers') + runbot_containers_memory = fields.Float('Memory limit for containers (in GiB)') + runbot_memory_bytes = fields.Float('Bytes', compute='_compute_memory_bytes') runbot_running_max = fields.Integer('Maximum number of running builds') runbot_timeout = fields.Integer('Max allowed step timeout (in seconds)') runbot_starting_port = fields.Integer('Starting port for running builds') @@ -40,6 +42,7 @@ class ResConfigSettings(models.TransientModel): res = super(ResConfigSettings, self).get_values() get_param = self.env['ir.config_parameter'].sudo().get_param res.update(runbot_workers=int(get_param('runbot.runbot_workers', default=2)), + runbot_containers_memory=float(get_param('runbot.runbot_containers_memory', default=0)), runbot_running_max=int(get_param('runbot.runbot_running_max', default=5)), runbot_timeout=int(get_param('runbot.runbot_timeout', default=10000)), runbot_starting_port=int(get_param('runbot.runbot_starting_port', default=2000)), @@ -59,6 +62,7 @@ class ResConfigSettings(models.TransientModel): super(ResConfigSettings, self).set_values() set_param = self.env['ir.config_parameter'].sudo().set_param set_param("runbot.runbot_workers", self.runbot_workers) + set_param("runbot.runbot_containers_memory", self.runbot_containers_memory) set_param("runbot.runbot_running_max", self.runbot_running_max) set_param("runbot.runbot_timeout", self.runbot_timeout) set_param("runbot.runbot_starting_port", self.runbot_starting_port) @@ -81,3 +85,11 @@ class ResConfigSettings(models.TransientModel): re.compile(self.runbot_is_base_regex) except re.error: raise UserError("The regex is invalid") + + @api.depends('runbot_containers_memory') + def _compute_memory_bytes(self): + for rec in self: + if rec.runbot_containers_memory > 0: + rec.runbot_memory_bytes = rec.runbot_containers_memory * 1024 ** 3 + else: + rec.runbot_memory_bytes = 0 diff --git a/runbot/views/res_config_settings_views.xml b/runbot/views/res_config_settings_views.xml index f7760623..4df48ed9 100644 --- a/runbot/views/res_config_settings_views.xml +++ b/runbot/views/res_config_settings_views.xml @@ -15,6 +15,9 @@