[IMP] runbot: limit memory usage of containers

In some conditions, it appears that a containerized build can eat up
all memory of the container host. This leads to disturbance of other
builds as the kernel OOM killer enters the dance.

With this commit, the docker ability to limit memory usage of a
container is used. The OOM killer will choose its victim among the
container processes.

The containers memory limit has to be set in the runbot settings. If not
set, no memory limit is used.

runbot/container.py

@@ -8,17 +8,13 @@ When testing this file:
     the first parameter should be a directory containing Odoo.
     The second parameter is the exposed port
 """
-import argparse
 import configparser
-import datetime
 import io
 import json
 import logging
 import os
 import re
-import shutil
 import subprocess
-import time
 
 
 _logger = logging.getLogger(__name__)
@@ -121,7 +117,7 @@ def docker_run(*args, **kwargs):
     return _docker_run(*args, **kwargs)
 
 
-def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False, image_tag=False, exposed_ports=None, cpu_limit=None, preexec_fn=None, ro_volumes=None, env_variables=None):
+def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False, image_tag=False, exposed_ports=None, cpu_limit=None, memory=None, preexec_fn=None, ro_volumes=None, env_variables=None):
     """Run tests in a docker container
     :param run_cmd: command string to run in container
     :param log_path: path to the logfile that will contain odoo stdout and stderr
@@ -130,6 +126,7 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False
     :param container_name: used to give a name to the container for later reference
     :param image_tag: Docker image tag name to select which docker image to use
     :param exposed_ports: if not None, starting at 8069, ports will be exposed as exposed_ports numbers
+    :param memory: memory limit in bytes for the container
     :params ro_volumes: dict of dest:source volumes to mount readonly in builddir
     :params env_variables: list of environment variables
     """
@@ -157,6 +154,10 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False
         '--shm-size=128m',
         '--init',
     ]
+
+    if memory:
+        docker_command.append('--memory=%s' % memory)
+
     if ro_volumes:
         for dest, source in ro_volumes.items():
             logs.write("Adding readonly volume '%s' pointing to %s \n" % (dest, source))

runbot/models/build.py

@@ -732,6 +732,9 @@ class BuildResult(models.Model):
             kwargs.update({'image_tag': self.params_id.dockerfile_id.image_tag})
         if kwargs['image_tag'] != 'odoo:DockerDefault':
             self._log('Preparing', 'Using Dockerfile Tag %s' % kwargs['image_tag'])
+        containers_memory_limit = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_containers_memory', 0)
+        if containers_memory_limit and 'memory' not in kwargs:
+            kwargs['memory'] = containers_memory_limit * 1024 ** 3
         docker_run(**kwargs)
 
     def _path(self, *l, **kw):

runbot/models/res_config_settings.py

@@ -10,6 +10,8 @@ class ResConfigSettings(models.TransientModel):
     _inherit = 'res.config.settings'
 
     runbot_workers = fields.Integer('Default number of workers')
+    runbot_containers_memory = fields.Float('Memory limit for containers (in GiB)')
+    runbot_memory_bytes = fields.Float('Bytes', compute='_compute_memory_bytes')
     runbot_running_max = fields.Integer('Maximum number of running builds')
     runbot_timeout = fields.Integer('Max allowed step timeout (in seconds)')
     runbot_starting_port = fields.Integer('Starting port for running builds')
@@ -40,6 +42,7 @@ class ResConfigSettings(models.TransientModel):
         res = super(ResConfigSettings, self).get_values()
         get_param = self.env['ir.config_parameter'].sudo().get_param
         res.update(runbot_workers=int(get_param('runbot.runbot_workers', default=2)),
+                   runbot_containers_memory=float(get_param('runbot.runbot_containers_memory', default=0)),
                    runbot_running_max=int(get_param('runbot.runbot_running_max', default=5)),
                    runbot_timeout=int(get_param('runbot.runbot_timeout', default=10000)),
                    runbot_starting_port=int(get_param('runbot.runbot_starting_port', default=2000)),
@@ -59,6 +62,7 @@ class ResConfigSettings(models.TransientModel):
         super(ResConfigSettings, self).set_values()
         set_param = self.env['ir.config_parameter'].sudo().set_param
         set_param("runbot.runbot_workers", self.runbot_workers)
+        set_param("runbot.runbot_containers_memory", self.runbot_containers_memory)
         set_param("runbot.runbot_running_max", self.runbot_running_max)
         set_param("runbot.runbot_timeout", self.runbot_timeout)
         set_param("runbot.runbot_starting_port", self.runbot_starting_port)
@@ -81,3 +85,11 @@ class ResConfigSettings(models.TransientModel):
                 re.compile(self.runbot_is_base_regex)
             except re.error:
                 raise UserError("The regex is invalid")
+
+    @api.depends('runbot_containers_memory')
+    def _compute_memory_bytes(self):
+        for rec in self:
+            if rec.runbot_containers_memory > 0:
+                rec.runbot_memory_bytes = rec.runbot_containers_memory * 1024 ** 3
+            else:
+                rec.runbot_memory_bytes = 0

runbot/views/res_config_settings_views.xml

@@ -15,6 +15,9 @@
                         <div class="content-group">
                           <label for="runbot_workers" class="col-xs-3 o_light_label" style="width: 60%;"/>
                           <field name="runbot_workers" style="width: 15%;"/>
+                          <label for="runbot_containers_memory" class="col-xs-3 o_light_label" style="width: 60%;"/>
+                          <field name="runbot_containers_memory" style="width: 15%;"/>&amp;nbsp;
+                          <field name="runbot_memory_bytes" readonly='1' style="width: 15%;"/>
                           <label for="runbot_running_max" class="col-xs-3 o_light_label" style="width: 60%;"/>
                           <field name="runbot_running_max" style="width: 15%;"/>
                           <label for="runbot_timeout" class="col-xs-3 o_light_label" style="width: 60%;"/>