[FIX] runbot: fix container cli tests

Since 81fefee, the container.py CLI does not work as expected.

With this commit, the CLI is working, a new arg was added to test
flamegraphs and the dump is adapted to mimic the runbot.

Also, a small issue is fixed in the zip file creation. Before the zip
creation, the directory is changed, if the directory change fails, the
zip is created from the current directory which is removed by zip at the
end of the process. That could lead to the deletion of the build dir.
This commit is contained in:
Christophe Monniez 2019-11-05 11:50:53 +01:00 committed by XavierDo
parent 685cc282ba
commit 631b29fcf8
2 changed files with 55 additions and 27 deletions

View File

@ -83,6 +83,7 @@ def docker_run(run_cmd, log_path, build_dir, container_name, exposed_ports=None,
:param container_name: used to give a name to the container for later reference
:param exposed_ports: if not None, starting at 8069, ports will be exposed as exposed_ports numbers
:params ro_volumes: dict of dest:source volumes to mount readonly in builddir
:params env_variables: list of environment variables
"""
_logger.debug('Docker run command: %s', run_cmd)
logs = open(log_path, 'w')
@ -159,6 +160,7 @@ def build(args):
logdir = os.path.join(args.build_dir, 'logs')
os.makedirs(logdir, exist_ok=True)
logfile = os.path.join(logdir, 'logs-build.txt')
_logger.info('Logfile is in %s', logfile)
docker_build(logfile, args.build_dir)
_logger.info('Finished building the base image container')
@ -176,14 +178,54 @@ def tests(args):
logfile = os.path.join(args.build_dir, 'logs', 'logs-build.txt')
docker_build(logfile, args.build_dir)
with open(os.path.join(args.build_dir, 'odoo-bin'), 'r') as exfile:
py_version = '3' if 'python3' in exfile.readline() else ''
# Test environment variables
if args.env:
cmd = Command(None, ['echo testa is $TESTA and testb is $TESTB '], None)
env_variables = ['TESTA=test a', 'TESTB="test b"']
env_log = os.path.join(args.build_dir, 'logs', 'logs-env.txt')
container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond
docker_run(cmd.build(), env_log, args.build_dir, container_name, env_variables=env_variables)
expected = 'testa is test a and testb is "test b"'
time.sleep(3) # ugly sleep to wait for docker process to flush the log file
assert expected in open(env_log,'r').read()
# Test testing
odoo_cmd = ['/data/build/odoo-bin', '-d %s' % args.db_name, '--addons-path=/data/build/addons', '--data-dir', '/data/build/datadir', '-r %s' % os.getlogin(), '-i', args.odoo_modules, '--test-enable', '--stop-after-init', '--max-cron-threads=0']
pres = [['sudo', 'pip%s' % py_version, 'install', '-r', '/data/build/requirements.txt']]
posts = None
python_params = []
if args.coverage:
omit = ['--omit', '*__manifest__.py']
python_params = [ '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + omit
posts = [['python%s' % py_version, "-m", "coverage", "html", "-d", "/data/build/coverage", "--ignore-errors"]]
os.makedirs(os.path.join(args.build_dir, 'coverage'), exist_ok=True)
elif args.flamegraph:
flame_log = '/data/build/logs/flame.log'
python_params = ['-m', 'flamegraph', '-o', flame_log]
odoo_cmd = ['python%s' % py_version ] + python_params + ['/data/build/odoo-bin', '-d %s' % args.db_name, '--addons-path=/data/build/addons', '--data-dir', '/data/build/datadir', '-r %s' % os.getlogin(), '-i', args.odoo_modules, '--test-enable', '--stop-after-init', '--max-cron-threads=0']
cmd = Command(pres, odoo_cmd, posts)
if args.dump:
os.makedirs(os.path.join(args.build_dir, 'logs', args.db_name), exist_ok=True)
dump_dir = '/data/build/logs/%s/' % args.db_name
sql_dest = '%s/dump.sql' % dump_dir
filestore_path = '/data/build/datadir/filestore/%s' % args.db_name
filestore_dest = '%s/filestore/' % dump_dir
zip_path = '/data/build/logs/%s.zip' % args.db_name
cmd.finals.append(['pg_dump', args.db_name, '>', sql_dest])
cmd.finals.append(['cp', '-r', filestore_path, filestore_dest])
cmd.finals.append(['cd', dump_dir, '&&', 'zip', '-rm9', zip_path, '*'])
if args.flamegraph:
cmd.finals.append(['flamegraph.pl', '--title', 'Flamegraph', flame_log, '>', '/data/build/logs/flame.svg'])
cmd.finals.append(['gzip', '-f', flame_log])
if args.kill:
logfile = os.path.join(args.build_dir, 'logs', 'logs-partial.txt')
container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond
# FIXME
# docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name)
docker_run(cmd.build(), logfile, args.build_dir, container_name)
# Test stopping the container
_logger.info('Waiting 30 sec before killing the build')
time.sleep(30)
@ -193,34 +235,18 @@ def tests(args):
# Test full testing
logfile = os.path.join(args.build_dir, 'logs', 'logs-full-test.txt')
container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond
if args.coverage:
omit = ['--omit', '*__manifest__.py']
with open(os.path.join(args.build_dir, 'odoo-bin'), 'r') as exfile:
pyversion = 'python3' if 'python3' in exfile.readline() else 'python'
odoo_cmd = [ pyversion, '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + omit + odoo_cmd
# FIXME
# docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name)
docker_run(cmd.build(), logfile, args.build_dir, container_name)
time.sleep(1) # give time for the container to start
while docker_is_running(container_name):
time.sleep(10)
_logger.info("Waiting for %s to stop", container_name)
if args.dump:
_logger.info("Testing pg_dump")
logfile = os.path.join(args.build_dir, 'logs', 'logs-pg_dump.txt')
container_name = 'odoo-container-test-pg_dump-%s' % datetime.datetime.now().microsecond
docker_pg_dump_cmd = 'cd /data/build/datadir && pg_dump -U %s -f db_export.sql %s' % (os.getlogin(), args.db_name)
docker_run(docker_pg_dump_cmd, logfile, args.build_dir, container_name)
time.sleep(1)
while docker_is_running(container_name):
time.sleep(10)
_logger.info("Waiting for %s to stop", container_name)
if args.run:
# Test running
logfile = os.path.join(args.build_dir, 'logs', 'logs-running.txt')
odoo_cmd = [
'python%s' % py_version,
'/data/build/odoo-bin', '-d %s' % args.db_name,
'--db-filter', '%s.*$' % args.db_name, '--addons-path=/data/build/addons',
'-r %s' % os.getlogin(), '-i', 'web', '--max-cron-threads=1',
@ -230,14 +256,14 @@ def tests(args):
if smtp_host:
odoo_cmd.extend(['--smtp', smtp_host])
container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond
cmd = Command(pres, odoo_cmd, [])
docker_run(cmd.build(), logfile, args.build_dir, container_name, exposed_ports=[args.odoo_port, args.odoo_port + 1], cpu_limit=300)
# FIXME
# docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name, exposed_ports=[args.odoo_port, args.odoo_port + 1], cpu_limit=300)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(name)s: %(message)s')
parser = argparse.ArgumentParser()
subparser = parser.add_subparsers(help='commands')
subparser = parser.add_subparsers(dest='command', required='True', help='commands')
p_build = subparser.add_parser('build', help='Build docker image')
p_build.add_argument('build_dir')
p_build.set_defaults(func=build)
@ -246,10 +272,13 @@ if __name__ == '__main__':
p_test.add_argument('build_dir')
p_test.add_argument('odoo_port', type=int)
p_test.add_argument('db_name')
p_test.add_argument('--coverage', action='store_true', help= 'test a build with coverage')
group = p_test.add_mutually_exclusive_group()
group.add_argument('--coverage', action='store_true', help= 'test a build with coverage')
group.add_argument('--flamegraph', action='store_true', help= 'test a build and draw a flamegraph')
p_test.add_argument('-i', dest='odoo_modules', default='web', help='Comma separated list of modules')
p_test.add_argument('--kill', action='store_true', default=False, help='Also test container kill')
p_test.add_argument('--dump', action='store_true', default=False, help='Test database export with pg_dump')
p_test.add_argument('--run', action='store_true', default=False, help='Also test running (Warning: the container survives exit)')
p_test.add_argument('--env', action='store_true', default=False, help='Test passing environment variables')
args = parser.parse_args()
args.func(args)

View File

@ -353,8 +353,7 @@ class ConfigStep(models.Model):
zip_path = '/data/build/logs/%s.zip' % db_name
cmd.finals.append(['pg_dump', db_name, '>', sql_dest])
cmd.finals.append(['cp', '-r', filestore_path, filestore_dest])
cmd.finals.append(['cd', dump_dir])
cmd.finals.append(['zip', '-rqm9', zip_path, '*'])
cmd.finals.append(['cd', dump_dir, '&&', 'zip', '-rmq9', zip_path, '*'])
infos = '{\n "db_name": "%s",\n "build_id": %s,\n "shas": [%s]\n}' % (db_name, build.id, ', '.join(['"%s"' % commit for commit in build._get_all_commit()]))
build.write_file('logs/%s/info.json' % db_name, infos)