diff --git a/runbot/container.py b/runbot/container.py index 7796e607..89053b51 100644 --- a/runbot/container.py +++ b/runbot/container.py @@ -122,7 +122,7 @@ def docker_run(*args, **kwargs): return _docker_run(*args, **kwargs) -def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False, image_tag=False, exposed_ports=None, cpu_limit=None, memory=None, preexec_fn=None, ro_volumes=None, env_variables=None): +def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False, image_tag=False, exposed_ports=None, cpu_limit=None, cpu_period=100000, cpus=0, memory=None, preexec_fn=None, ro_volumes=None, env_variables=None): """Run tests in a docker container :param run_cmd: command string to run in container :param log_path: path to the logfile that will contain odoo stdout and stderr @@ -131,6 +131,8 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False :param container_name: used to give a name to the container for later reference :param image_tag: Docker image tag name to select which docker image to use :param exposed_ports: if not None, starting at 8069, ports will be exposed as exposed_ports numbers + :param cpu_period: Specify the CPU CFS scheduler period, which is used alongside cpu_quota + :param cpus: used to compute cpu_quota = cpu_period * cpus (equivalent of --cpus in docker CLI) :param memory: memory limit in bytes for the container :params ro_volumes: dict of dest:source volumes to mount readonly in builddir :params env_variables: list of environment variables @@ -181,6 +183,8 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False mem_limit=memory, ports=ports, ulimits=ulimits, + cpu_period=cpu_period, + cpu_quota=int(cpus * cpu_period ) if cpus else None, environment=env_variables, init=True, command=['/bin/bash', '-c', diff --git a/runbot/models/build.py b/runbot/models/build.py index a2355ffa..3e5c42f2 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -770,6 +770,7 @@ class BuildResult(models.Model): containers_memory_limit = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_containers_memory', 0) if containers_memory_limit and 'memory' not in kwargs: kwargs['memory'] = int(float(containers_memory_limit) * 1024 ** 3) + self.docker_start = now() if self.job_start: start_step_time = int(dt2time(self.docker_start) - dt2time(self.job_start)) diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index f87c9f8b..30840efb 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -3,6 +3,7 @@ import glob import json import logging import fnmatch +import psutil import re import shlex import time @@ -154,6 +155,7 @@ class ConfigStep(models.Model): install_modules = fields.Char('Modules to install', help="List of module patterns to install, use * to install all available modules, prefix the pattern with dash to remove the module.", default='') db_name = fields.Char('Db Name', compute='_compute_db_name', inverse='_inverse_db_name', tracking=True) cpu_limit = fields.Integer('Cpu limit', default=3600, tracking=True) + container_cpus = fields.Integer('Allowed CPUs', help='Allowed container CPUs. Fallback on config parameter if 0.', default=0, tracking=True) coverage = fields.Boolean('Coverage', default=False, tracking=True) paths_to_omit = fields.Char('Paths to omit from coverage', tracking=True) flamegraph = fields.Boolean('Allow Flamegraph', default=False, tracking=True) @@ -282,6 +284,11 @@ class ConfigStep(models.Model): run_method = getattr(self, '_run_%s' % self.job_type) docker_params = run_method(build, log_path, **kwargs) if docker_params: + container_cpus = float(self.container_cpus or self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_containers_cpus', 0)) + if 'cpus' not in docker_params and container_cpus: + logical_cpu_count = psutil.cpu_count(logical=True) + physical_cpu_count = psutil.cpu_count(logical=False) + docker_params['cpus'] = float((logical_cpu_count / physical_cpu_count) * container_cpus) build._docker_run(**docker_params) def _run_create_build(self, build, log_path): diff --git a/runbot/models/res_config_settings.py b/runbot/models/res_config_settings.py index 196264f1..1b62e2df 100644 --- a/runbot/models/res_config_settings.py +++ b/runbot/models/res_config_settings.py @@ -10,6 +10,7 @@ class ResConfigSettings(models.TransientModel): _inherit = 'res.config.settings' runbot_workers = fields.Integer('Default number of workers') + runbot_containers_cpus = fields.Float('Allowed Containers CPUs (0 means no limit)') runbot_containers_memory = fields.Float('Memory limit for containers (in GiB)') runbot_memory_bytes = fields.Float('Bytes', compute='_compute_memory_bytes') runbot_running_max = fields.Integer('Maximum number of running builds') @@ -58,6 +59,7 @@ class ResConfigSettings(models.TransientModel): res = super(ResConfigSettings, self).get_values() get_param = self.env['ir.config_parameter'].sudo().get_param res.update(runbot_workers=int(get_param('runbot.runbot_workers', default=2)), + runbot_containers_cpus=float(get_param('runbot.runbot_containers_cpus', default=0)), runbot_containers_memory=float(get_param('runbot.runbot_containers_memory', default=0)), runbot_running_max=int(get_param('runbot.runbot_running_max', default=5)), runbot_timeout=int(get_param('runbot.runbot_timeout', default=10000)), @@ -81,6 +83,7 @@ class ResConfigSettings(models.TransientModel): super(ResConfigSettings, self).set_values() set_param = self.env['ir.config_parameter'].sudo().set_param set_param("runbot.runbot_workers", self.runbot_workers) + set_param("runbot.runbot_containers_cpus", self.runbot_containers_cpus) set_param("runbot.runbot_containers_memory", self.runbot_containers_memory) set_param("runbot.runbot_running_max", self.runbot_running_max) set_param("runbot.runbot_timeout", self.runbot_timeout) diff --git a/runbot/views/res_config_settings_views.xml b/runbot/views/res_config_settings_views.xml index 18603298..825b570c 100644 --- a/runbot/views/res_config_settings_views.xml +++ b/runbot/views/res_config_settings_views.xml @@ -16,6 +16,8 @@