diff --git a/runbot/container.py b/runbot/container.py index 73b1da27..35ebe854 100644 --- a/runbot/container.py +++ b/runbot/container.py @@ -30,6 +30,20 @@ USER odoo ENV COVERAGE_FILE /data/build/.coverage """ % {'group_id': os.getgid(), 'user_id': os.getuid()} + +def build_odoo_cmd(odoo_cmd): + """ returns the chain of commands necessary to run odoo inside the container + : param odoo_cmd: odoo command as a list + : returns: a string with the command chain to execute in the docker container + """ + # build cmd + cmd_chain = [] + cmd_chain.append('cd /data/build') + cmd_chain.append('head -1 odoo-bin | grep -q python3 && sudo pip3 install -r requirements.txt || sudo pip install -r requirements.txt') + cmd_chain.append(' '.join(odoo_cmd)) + return ' && '.join(cmd_chain) + + def docker_build(log_path, build_dir): """Build the docker image :param log_path: path to the logfile that will contain odoo stdout and stderr @@ -46,21 +60,15 @@ def docker_build(log_path, build_dir): dbuild = subprocess.Popen(['docker', 'build', '--tag', 'odoo:runbot_tests', '.'], stdout=logs, stderr=logs, cwd=docker_dir) dbuild.wait() -def docker_run(odoo_cmd, log_path, build_dir, container_name, exposed_ports=None, cpu_limit=None, preexec_fn=None): +def docker_run(run_cmd, log_path, build_dir, container_name, exposed_ports=None, cpu_limit=None, preexec_fn=None): """Run tests in a docker container - :param odoo_cmd: command that starts odoo + :param run_cmd: command string to run in container :param log_path: path to the logfile that will contain odoo stdout and stderr :param build_dir: the build directory that contains the Odoo sources to build. This directory is shared as a volume with the container :param container_name: used to give a name to the container for later reference :param exposed_ports: if not None, starting at 8069, ports will be exposed as exposed_ports numbers """ - # build cmd - cmd_chain = [] - cmd_chain.append('cd /data/build') - cmd_chain.append('head -1 odoo-bin | grep -q python3 && sudo pip3 install -r requirements.txt || sudo pip install -r requirements.txt') - cmd_chain.append(' '.join(odoo_cmd)) - run_cmd = ' && '.join(cmd_chain) _logger.debug('Docker run command: %s', run_cmd) logs = open(log_path, 'w') @@ -138,7 +146,7 @@ def tests(args): if args.kill: logfile = os.path.join(args.build_dir, 'logs', 'logs-partial.txt') container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond - docker_run(odoo_cmd, logfile, args.build_dir, container_name) + docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name) # Test stopping the container _logger.info('Waiting 30 sec before killing the build') time.sleep(30) @@ -153,13 +161,24 @@ def tests(args): with open(os.path.join(args.build_dir, 'odoo-bin'), 'r') as exfile: pyversion = 'python3' if 'python3' in exfile.readline() else 'python' odoo_cmd = [ pyversion, '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + omit + odoo_cmd - docker_run(odoo_cmd, logfile, args.build_dir, container_name) + docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name) time.sleep(1) # give time for the container to start while docker_is_running(container_name): time.sleep(10) _logger.info("Waiting for %s to stop", container_name) + if args.dump: + _logger.info("Testing pg_dump") + logfile = os.path.join(args.build_dir, 'logs', 'logs-pg_dump.txt') + container_name = 'odoo-container-test-pg_dump-%s' % datetime.datetime.now().microsecond + docker_pg_dump_cmd = 'cd /data/build/datadir && pg_dump -U %s -f db_export.sql %s' % (os.getlogin(), args.db_name) + docker_run(docker_pg_dump_cmd, logfile, args.build_dir, container_name) + time.sleep(1) + while docker_is_running(container_name): + time.sleep(10) + _logger.info("Waiting for %s to stop", container_name) + if args.run: # Test running logfile = os.path.join(args.build_dir, 'logs', 'logs-running.txt') @@ -173,7 +192,7 @@ def tests(args): if smtp_host: odoo_cmd.extend(['--smtp', smtp_host]) container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond - docker_run(odoo_cmd, logfile, args.build_dir, container_name, exposed_ports=[args.odoo_port, args.odoo_port + 1], cpu_limit=300) + docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name, exposed_ports=[args.odoo_port, args.odoo_port + 1], cpu_limit=300) if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(name)s: %(message)s') @@ -190,6 +209,7 @@ if __name__ == '__main__': p_test.add_argument('--coverage', action='store_true', help= 'test a build with coverage') p_test.add_argument('-i', dest='odoo_modules', default='web', help='Comma separated list of modules') p_test.add_argument('--kill', action='store_true', default=False, help='Also test container kill') + p_test.add_argument('--dump', action='store_true', default=False, help='Test database export with pg_dump') p_test.add_argument('--run', action='store_true', default=False, help='Also test running (Warning: the container survives exit)') args = parser.parse_args() args.func(args) diff --git a/runbot/models/build.py b/runbot/models/build.py index 8a4abfb4..d0850e24 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -12,7 +12,7 @@ import subprocess import time from subprocess import CalledProcessError from ..common import dt2time, fqdn, now, grep, time2str, rfind, uniq_list, local_pgadmin_cursor, get_py_version -from ..container import docker_build, docker_run, docker_stop, docker_is_running, docker_get_gateway_ip +from ..container import docker_build, docker_run, docker_stop, docker_is_running, docker_get_gateway_ip, build_odoo_cmd from odoo import models, fields, api from odoo.exceptions import UserError from odoo.http import request @@ -789,7 +789,7 @@ class runbot_build(models.Model): cmd += ['-d', '%s-base' % build.dest, '-i', 'base', '--stop-after-init', '--log-level=test', '--max-cron-threads=0'] if build.extra_params: cmd.extend(shlex.split(build.extra_params)) - return docker_run(cmd, log_path, build._path(), build._get_docker_name(), cpu_limit=600) + return docker_run(build_odoo_cmd(cmd), log_path, build._path(), build._get_docker_name(), cpu_limit=600) @runbot_job('testing', 'running') def _job_20_test_all(self, build, log_path): @@ -817,7 +817,7 @@ class runbot_build(models.Model): cmd = [ get_py_version(build), '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + omit + cmd # reset job_start to an accurate job_20 job_time build.write({'job_start': now()}) - return docker_run(cmd, log_path, build._path(), build._get_docker_name(), cpu_limit=cpu_limit) + return docker_run(build_odoo_cmd(cmd), log_path, build._path(), build._get_docker_name(), cpu_limit=cpu_limit) @runbot_job('testing') def _job_21_coverage_html(self, build, log_path): @@ -827,7 +827,7 @@ class runbot_build(models.Model): cov_path = build._path('coverage') os.makedirs(cov_path, exist_ok=True) cmd = [ get_py_version(build), "-m", "coverage", "html", "-d", "/data/build/coverage", "--ignore-errors"] - return docker_run(cmd, log_path, build._path(), build._get_docker_name()) + return docker_run(build_odoo_cmd(cmd), log_path, build._path(), build._get_docker_name()) @runbot_job('testing') def _job_22_coverage_result(self, build, log_path): @@ -889,4 +889,4 @@ class runbot_build(models.Model): smtp_host = docker_get_gateway_ip() if smtp_host: cmd += ['--smtp', smtp_host] - return docker_run(cmd, log_path, build._path(), build._get_docker_name(), exposed_ports = [build.port, build.port + 1]) + return docker_run(build_odoo_cmd(cmd), log_path, build._path(), build._get_docker_name(), exposed_ports = [build.port, build.port + 1])