mirror of
https://github.com/odoo/runbot.git
synced 2025-03-17 08:25:46 +07:00
[FIX] runbot: fix container cli tests
Since 81fefee, the container.py CLI does not work as expected.
With this commit, the CLI is working, a new arg was added to test
flamegraphs and the dump is adapted to mimic the runbot.
Also, a small issue is fixed in the zip file creation. Before the zip
creation, the directory is changed, if the directory change fails, the
zip is created from the current directory which is removed by zip at the
end of the process. That could lead to the deletion of the build dir.
This commit is contained in:
Christophe Monniez
XavierDo
parent
685cc282ba
commit
631b29fcf8
@ -83,6 +83,7 @@ def docker_run(run_cmd, log_path, build_dir, container_name, exposed_ports=None,
|
||||
:param container_name: used to give a name to the container for later reference
|
||||
:param exposed_ports: if not None, starting at 8069, ports will be exposed as exposed_ports numbers
|
||||
:params ro_volumes: dict of dest:source volumes to mount readonly in builddir
|
||||
:params env_variables: list of environment variables
|
||||
"""
|
||||
_logger.debug('Docker run command: %s', run_cmd)
|
||||
logs = open(log_path, 'w')
|
||||
@ -159,6 +160,7 @@ def build(args):
|
||||
logdir = os.path.join(args.build_dir, 'logs')
|
||||
os.makedirs(logdir, exist_ok=True)
|
||||
logfile = os.path.join(logdir, 'logs-build.txt')
|
||||
_logger.info('Logfile is in %s', logfile)
|
||||
docker_build(logfile, args.build_dir)
|
||||
_logger.info('Finished building the base image container')
|
||||
|
||||
@ -176,14 +178,54 @@ def tests(args):
|
||||
logfile = os.path.join(args.build_dir, 'logs', 'logs-build.txt')
|
||||
docker_build(logfile, args.build_dir)
|
||||
|
||||
with open(os.path.join(args.build_dir, 'odoo-bin'), 'r') as exfile:
|
||||
py_version = '3' if 'python3' in exfile.readline() else ''
|
||||
|
||||
# Test environment variables
|
||||
if args.env:
|
||||
cmd = Command(None, ['echo testa is $TESTA and testb is $TESTB '], None)
|
||||
env_variables = ['TESTA=test a', 'TESTB="test b"']
|
||||
env_log = os.path.join(args.build_dir, 'logs', 'logs-env.txt')
|
||||
container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond
|
||||
docker_run(cmd.build(), env_log, args.build_dir, container_name, env_variables=env_variables)
|
||||
expected = 'testa is test a and testb is "test b"'
|
||||
time.sleep(3) # ugly sleep to wait for docker process to flush the log file
|
||||
assert expected in open(env_log,'r').read()
|
||||
|
||||
# Test testing
|
||||
odoo_cmd = ['/data/build/odoo-bin', '-d %s' % args.db_name, '--addons-path=/data/build/addons', '--data-dir', '/data/build/datadir', '-r %s' % os.getlogin(), '-i', args.odoo_modules, '--test-enable', '--stop-after-init', '--max-cron-threads=0']
|
||||
pres = [['sudo', 'pip%s' % py_version, 'install', '-r', '/data/build/requirements.txt']]
|
||||
posts = None
|
||||
python_params = []
|
||||
if args.coverage:
|
||||
omit = ['--omit', '*__manifest__.py']
|
||||
python_params = [ '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + omit
|
||||
posts = [['python%s' % py_version, "-m", "coverage", "html", "-d", "/data/build/coverage", "--ignore-errors"]]
|
||||
os.makedirs(os.path.join(args.build_dir, 'coverage'), exist_ok=True)
|
||||
elif args.flamegraph:
|
||||
flame_log = '/data/build/logs/flame.log'
|
||||
python_params = ['-m', 'flamegraph', '-o', flame_log]
|
||||
odoo_cmd = ['python%s' % py_version ] + python_params + ['/data/build/odoo-bin', '-d %s' % args.db_name, '--addons-path=/data/build/addons', '--data-dir', '/data/build/datadir', '-r %s' % os.getlogin(), '-i', args.odoo_modules, '--test-enable', '--stop-after-init', '--max-cron-threads=0']
|
||||
cmd = Command(pres, odoo_cmd, posts)
|
||||
|
||||
if args.dump:
|
||||
os.makedirs(os.path.join(args.build_dir, 'logs', args.db_name), exist_ok=True)
|
||||
dump_dir = '/data/build/logs/%s/' % args.db_name
|
||||
sql_dest = '%s/dump.sql' % dump_dir
|
||||
filestore_path = '/data/build/datadir/filestore/%s' % args.db_name
|
||||
filestore_dest = '%s/filestore/' % dump_dir
|
||||
zip_path = '/data/build/logs/%s.zip' % args.db_name
|
||||
cmd.finals.append(['pg_dump', args.db_name, '>', sql_dest])
|
||||
cmd.finals.append(['cp', '-r', filestore_path, filestore_dest])
|
||||
cmd.finals.append(['cd', dump_dir, '&&', 'zip', '-rm9', zip_path, '*'])
|
||||
|
||||
if args.flamegraph:
|
||||
cmd.finals.append(['flamegraph.pl', '--title', 'Flamegraph', flame_log, '>', '/data/build/logs/flame.svg'])
|
||||
cmd.finals.append(['gzip', '-f', flame_log])
|
||||
|
||||
if args.kill:
|
||||
logfile = os.path.join(args.build_dir, 'logs', 'logs-partial.txt')
|
||||
container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond
|
||||
# FIXME
|
||||
# docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name)
|
||||
docker_run(cmd.build(), logfile, args.build_dir, container_name)
|
||||
# Test stopping the container
|
||||
_logger.info('Waiting 30 sec before killing the build')
|
||||
time.sleep(30)
|
||||
@ -193,34 +235,18 @@ def tests(args):
|
||||
# Test full testing
|
||||
logfile = os.path.join(args.build_dir, 'logs', 'logs-full-test.txt')
|
||||
container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond
|
||||
if args.coverage:
|
||||
omit = ['--omit', '*__manifest__.py']
|
||||
with open(os.path.join(args.build_dir, 'odoo-bin'), 'r') as exfile:
|
||||
pyversion = 'python3' if 'python3' in exfile.readline() else 'python'
|
||||
odoo_cmd = [ pyversion, '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + omit + odoo_cmd
|
||||
# FIXME
|
||||
# docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name)
|
||||
docker_run(cmd.build(), logfile, args.build_dir, container_name)
|
||||
time.sleep(1) # give time for the container to start
|
||||
|
||||
while docker_is_running(container_name):
|
||||
time.sleep(10)
|
||||
_logger.info("Waiting for %s to stop", container_name)
|
||||
|
||||
if args.dump:
|
||||
_logger.info("Testing pg_dump")
|
||||
logfile = os.path.join(args.build_dir, 'logs', 'logs-pg_dump.txt')
|
||||
container_name = 'odoo-container-test-pg_dump-%s' % datetime.datetime.now().microsecond
|
||||
docker_pg_dump_cmd = 'cd /data/build/datadir && pg_dump -U %s -f db_export.sql %s' % (os.getlogin(), args.db_name)
|
||||
docker_run(docker_pg_dump_cmd, logfile, args.build_dir, container_name)
|
||||
time.sleep(1)
|
||||
while docker_is_running(container_name):
|
||||
time.sleep(10)
|
||||
_logger.info("Waiting for %s to stop", container_name)
|
||||
|
||||
if args.run:
|
||||
# Test running
|
||||
logfile = os.path.join(args.build_dir, 'logs', 'logs-running.txt')
|
||||
odoo_cmd = [
|
||||
'python%s' % py_version,
|
||||
'/data/build/odoo-bin', '-d %s' % args.db_name,
|
||||
'--db-filter', '%s.*$' % args.db_name, '--addons-path=/data/build/addons',
|
||||
'-r %s' % os.getlogin(), '-i', 'web', '--max-cron-threads=1',
|
||||
@ -230,14 +256,14 @@ def tests(args):
|
||||
if smtp_host:
|
||||
odoo_cmd.extend(['--smtp', smtp_host])
|
||||
container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond
|
||||
cmd = Command(pres, odoo_cmd, [])
|
||||
docker_run(cmd.build(), logfile, args.build_dir, container_name, exposed_ports=[args.odoo_port, args.odoo_port + 1], cpu_limit=300)
|
||||
|
||||
# FIXME
|
||||
# docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name, exposed_ports=[args.odoo_port, args.odoo_port + 1], cpu_limit=300)
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(name)s: %(message)s')
|
||||
parser = argparse.ArgumentParser()
|
||||
subparser = parser.add_subparsers(help='commands')
|
||||
subparser = parser.add_subparsers(dest='command', required='True', help='commands')
|
||||
p_build = subparser.add_parser('build', help='Build docker image')
|
||||
p_build.add_argument('build_dir')
|
||||
p_build.set_defaults(func=build)
|
||||
@ -246,10 +272,13 @@ if __name__ == '__main__':
|
||||
p_test.add_argument('build_dir')
|
||||
p_test.add_argument('odoo_port', type=int)
|
||||
p_test.add_argument('db_name')
|
||||
p_test.add_argument('--coverage', action='store_true', help= 'test a build with coverage')
|
||||
group = p_test.add_mutually_exclusive_group()
|
||||
group.add_argument('--coverage', action='store_true', help= 'test a build with coverage')
|
||||
group.add_argument('--flamegraph', action='store_true', help= 'test a build and draw a flamegraph')
|
||||
p_test.add_argument('-i', dest='odoo_modules', default='web', help='Comma separated list of modules')
|
||||
p_test.add_argument('--kill', action='store_true', default=False, help='Also test container kill')
|
||||
p_test.add_argument('--dump', action='store_true', default=False, help='Test database export with pg_dump')
|
||||
p_test.add_argument('--run', action='store_true', default=False, help='Also test running (Warning: the container survives exit)')
|
||||
p_test.add_argument('--env', action='store_true', default=False, help='Test passing environment variables')
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
|
||||
@ -353,8 +353,7 @@ class ConfigStep(models.Model):
|
||||
zip_path = '/data/build/logs/%s.zip' % db_name
|
||||
cmd.finals.append(['pg_dump', db_name, '>', sql_dest])
|
||||
cmd.finals.append(['cp', '-r', filestore_path, filestore_dest])
|
||||
cmd.finals.append(['cd', dump_dir])
|
||||
cmd.finals.append(['zip', '-rqm9', zip_path, '*'])
|
||||
cmd.finals.append(['cd', dump_dir, '&&', 'zip', '-rmq9', zip_path, '*'])
|
||||
infos = '{\n "db_name": "%s",\n "build_id": %s,\n "shas": [%s]\n}' % (db_name, build.id, ', '.join(['"%s"' % commit for commit in build._get_all_commit()]))
|
||||
build.write_file('logs/%s/info.json' % db_name, infos)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user