[IMP] runbot: limit memory usage of containers

In some conditions, it appears that a containerized build can eat up
all memory of the container host. This leads to disturbance of other
builds as the kernel OOM killer enters the dance.

With this commit, the docker ability to limit memory usage of a
container is used. The OOM killer will choose its victim among the
container processes.

The containers memory limit has to be set in the runbot settings. If not
set, no memory limit is used.
This commit is contained in:
Christophe Monniez 2021-06-30 15:45:41 +02:00 committed by xdo
parent 4b16e889eb
commit 847622552f
4 changed files with 24 additions and 5 deletions

View File

@ -8,17 +8,13 @@ When testing this file:
the first parameter should be a directory containing Odoo.
The second parameter is the exposed port
"""
import argparse
import configparser
import datetime
import io
import json
import logging
import os
import re
import shutil
import subprocess
import time
_logger = logging.getLogger(__name__)
@ -121,7 +117,7 @@ def docker_run(*args, **kwargs):
return _docker_run(*args, **kwargs)
def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False, image_tag=False, exposed_ports=None, cpu_limit=None, preexec_fn=None, ro_volumes=None, env_variables=None):
def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False, image_tag=False, exposed_ports=None, cpu_limit=None, memory=None, preexec_fn=None, ro_volumes=None, env_variables=None):
"""Run tests in a docker container
:param run_cmd: command string to run in container
:param log_path: path to the logfile that will contain odoo stdout and stderr
@ -130,6 +126,7 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False
:param container_name: used to give a name to the container for later reference
:param image_tag: Docker image tag name to select which docker image to use
:param exposed_ports: if not None, starting at 8069, ports will be exposed as exposed_ports numbers
:param memory: memory limit in bytes for the container
:params ro_volumes: dict of dest:source volumes to mount readonly in builddir
:params env_variables: list of environment variables
"""
@ -157,6 +154,10 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False
'--shm-size=128m',
'--init',
]
if memory:
docker_command.append('--memory=%s' % memory)
if ro_volumes:
for dest, source in ro_volumes.items():
logs.write("Adding readonly volume '%s' pointing to %s \n" % (dest, source))

View File

@ -732,6 +732,9 @@ class BuildResult(models.Model):
kwargs.update({'image_tag': self.params_id.dockerfile_id.image_tag})
if kwargs['image_tag'] != 'odoo:DockerDefault':
self._log('Preparing', 'Using Dockerfile Tag %s' % kwargs['image_tag'])
containers_memory_limit = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_containers_memory', 0)
if containers_memory_limit and 'memory' not in kwargs:
kwargs['memory'] = containers_memory_limit * 1024 ** 3
docker_run(**kwargs)
def _path(self, *l, **kw):

View File

@ -10,6 +10,8 @@ class ResConfigSettings(models.TransientModel):
_inherit = 'res.config.settings'
runbot_workers = fields.Integer('Default number of workers')
runbot_containers_memory = fields.Float('Memory limit for containers (in GiB)')
runbot_memory_bytes = fields.Float('Bytes', compute='_compute_memory_bytes')
runbot_running_max = fields.Integer('Maximum number of running builds')
runbot_timeout = fields.Integer('Max allowed step timeout (in seconds)')
runbot_starting_port = fields.Integer('Starting port for running builds')
@ -40,6 +42,7 @@ class ResConfigSettings(models.TransientModel):
res = super(ResConfigSettings, self).get_values()
get_param = self.env['ir.config_parameter'].sudo().get_param
res.update(runbot_workers=int(get_param('runbot.runbot_workers', default=2)),
runbot_containers_memory=float(get_param('runbot.runbot_containers_memory', default=0)),
runbot_running_max=int(get_param('runbot.runbot_running_max', default=5)),
runbot_timeout=int(get_param('runbot.runbot_timeout', default=10000)),
runbot_starting_port=int(get_param('runbot.runbot_starting_port', default=2000)),
@ -59,6 +62,7 @@ class ResConfigSettings(models.TransientModel):
super(ResConfigSettings, self).set_values()
set_param = self.env['ir.config_parameter'].sudo().set_param
set_param("runbot.runbot_workers", self.runbot_workers)
set_param("runbot.runbot_containers_memory", self.runbot_containers_memory)
set_param("runbot.runbot_running_max", self.runbot_running_max)
set_param("runbot.runbot_timeout", self.runbot_timeout)
set_param("runbot.runbot_starting_port", self.runbot_starting_port)
@ -81,3 +85,11 @@ class ResConfigSettings(models.TransientModel):
re.compile(self.runbot_is_base_regex)
except re.error:
raise UserError("The regex is invalid")
@api.depends('runbot_containers_memory')
def _compute_memory_bytes(self):
for rec in self:
if rec.runbot_containers_memory > 0:
rec.runbot_memory_bytes = rec.runbot_containers_memory * 1024 ** 3
else:
rec.runbot_memory_bytes = 0

View File

@ -15,6 +15,9 @@
<div class="content-group">
<label for="runbot_workers" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_workers" style="width: 15%;"/>
<label for="runbot_containers_memory" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_containers_memory" style="width: 15%;"/>&amp;nbsp;
<field name="runbot_memory_bytes" readonly='1' style="width: 15%;"/>
<label for="runbot_running_max" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_running_max" style="width: 15%;"/>
<label for="runbot_timeout" class="col-xs-3 o_light_label" style="width: 60%;"/>