From 631b29fcf8c3634f72501a5fc8760953fe70e194 Mon Sep 17 00:00:00 2001 From: Christophe Monniez Date: Tue, 5 Nov 2019 11:50:53 +0100 Subject: [PATCH] [FIX] runbot: fix container cli tests Since 81fefee, the container.py CLI does not work as expected. With this commit, the CLI is working, a new arg was added to test flamegraphs and the dump is adapted to mimic the runbot. Also, a small issue is fixed in the zip file creation. Before the zip creation, the directory is changed, if the directory change fails, the zip is created from the current directory which is removed by zip at the end of the process. That could lead to the deletion of the build dir. --- runbot/container.py | 79 ++++++++++++++++++++++++----------- runbot/models/build_config.py | 3 +- 2 files changed, 55 insertions(+), 27 deletions(-) diff --git a/runbot/container.py b/runbot/container.py index 22ea097a..bc27bfea 100644 --- a/runbot/container.py +++ b/runbot/container.py @@ -83,6 +83,7 @@ def docker_run(run_cmd, log_path, build_dir, container_name, exposed_ports=None, :param container_name: used to give a name to the container for later reference :param exposed_ports: if not None, starting at 8069, ports will be exposed as exposed_ports numbers :params ro_volumes: dict of dest:source volumes to mount readonly in builddir + :params env_variables: list of environment variables """ _logger.debug('Docker run command: %s', run_cmd) logs = open(log_path, 'w') @@ -159,6 +160,7 @@ def build(args): logdir = os.path.join(args.build_dir, 'logs') os.makedirs(logdir, exist_ok=True) logfile = os.path.join(logdir, 'logs-build.txt') + _logger.info('Logfile is in %s', logfile) docker_build(logfile, args.build_dir) _logger.info('Finished building the base image container') @@ -176,14 +178,54 @@ def tests(args): logfile = os.path.join(args.build_dir, 'logs', 'logs-build.txt') docker_build(logfile, args.build_dir) + with open(os.path.join(args.build_dir, 'odoo-bin'), 'r') as exfile: + py_version = '3' if 'python3' in exfile.readline() else '' + + # Test environment variables + if args.env: + cmd = Command(None, ['echo testa is $TESTA and testb is $TESTB '], None) + env_variables = ['TESTA=test a', 'TESTB="test b"'] + env_log = os.path.join(args.build_dir, 'logs', 'logs-env.txt') + container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond + docker_run(cmd.build(), env_log, args.build_dir, container_name, env_variables=env_variables) + expected = 'testa is test a and testb is "test b"' + time.sleep(3) # ugly sleep to wait for docker process to flush the log file + assert expected in open(env_log,'r').read() + # Test testing - odoo_cmd = ['/data/build/odoo-bin', '-d %s' % args.db_name, '--addons-path=/data/build/addons', '--data-dir', '/data/build/datadir', '-r %s' % os.getlogin(), '-i', args.odoo_modules, '--test-enable', '--stop-after-init', '--max-cron-threads=0'] + pres = [['sudo', 'pip%s' % py_version, 'install', '-r', '/data/build/requirements.txt']] + posts = None + python_params = [] + if args.coverage: + omit = ['--omit', '*__manifest__.py'] + python_params = [ '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + omit + posts = [['python%s' % py_version, "-m", "coverage", "html", "-d", "/data/build/coverage", "--ignore-errors"]] + os.makedirs(os.path.join(args.build_dir, 'coverage'), exist_ok=True) + elif args.flamegraph: + flame_log = '/data/build/logs/flame.log' + python_params = ['-m', 'flamegraph', '-o', flame_log] + odoo_cmd = ['python%s' % py_version ] + python_params + ['/data/build/odoo-bin', '-d %s' % args.db_name, '--addons-path=/data/build/addons', '--data-dir', '/data/build/datadir', '-r %s' % os.getlogin(), '-i', args.odoo_modules, '--test-enable', '--stop-after-init', '--max-cron-threads=0'] + cmd = Command(pres, odoo_cmd, posts) + + if args.dump: + os.makedirs(os.path.join(args.build_dir, 'logs', args.db_name), exist_ok=True) + dump_dir = '/data/build/logs/%s/' % args.db_name + sql_dest = '%s/dump.sql' % dump_dir + filestore_path = '/data/build/datadir/filestore/%s' % args.db_name + filestore_dest = '%s/filestore/' % dump_dir + zip_path = '/data/build/logs/%s.zip' % args.db_name + cmd.finals.append(['pg_dump', args.db_name, '>', sql_dest]) + cmd.finals.append(['cp', '-r', filestore_path, filestore_dest]) + cmd.finals.append(['cd', dump_dir, '&&', 'zip', '-rm9', zip_path, '*']) + + if args.flamegraph: + cmd.finals.append(['flamegraph.pl', '--title', 'Flamegraph', flame_log, '>', '/data/build/logs/flame.svg']) + cmd.finals.append(['gzip', '-f', flame_log]) if args.kill: logfile = os.path.join(args.build_dir, 'logs', 'logs-partial.txt') container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond - # FIXME - # docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name) + docker_run(cmd.build(), logfile, args.build_dir, container_name) # Test stopping the container _logger.info('Waiting 30 sec before killing the build') time.sleep(30) @@ -193,34 +235,18 @@ def tests(args): # Test full testing logfile = os.path.join(args.build_dir, 'logs', 'logs-full-test.txt') container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond - if args.coverage: - omit = ['--omit', '*__manifest__.py'] - with open(os.path.join(args.build_dir, 'odoo-bin'), 'r') as exfile: - pyversion = 'python3' if 'python3' in exfile.readline() else 'python' - odoo_cmd = [ pyversion, '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + omit + odoo_cmd - # FIXME - # docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name) + docker_run(cmd.build(), logfile, args.build_dir, container_name) time.sleep(1) # give time for the container to start while docker_is_running(container_name): time.sleep(10) _logger.info("Waiting for %s to stop", container_name) - if args.dump: - _logger.info("Testing pg_dump") - logfile = os.path.join(args.build_dir, 'logs', 'logs-pg_dump.txt') - container_name = 'odoo-container-test-pg_dump-%s' % datetime.datetime.now().microsecond - docker_pg_dump_cmd = 'cd /data/build/datadir && pg_dump -U %s -f db_export.sql %s' % (os.getlogin(), args.db_name) - docker_run(docker_pg_dump_cmd, logfile, args.build_dir, container_name) - time.sleep(1) - while docker_is_running(container_name): - time.sleep(10) - _logger.info("Waiting for %s to stop", container_name) - if args.run: # Test running logfile = os.path.join(args.build_dir, 'logs', 'logs-running.txt') odoo_cmd = [ + 'python%s' % py_version, '/data/build/odoo-bin', '-d %s' % args.db_name, '--db-filter', '%s.*$' % args.db_name, '--addons-path=/data/build/addons', '-r %s' % os.getlogin(), '-i', 'web', '--max-cron-threads=1', @@ -230,14 +256,14 @@ def tests(args): if smtp_host: odoo_cmd.extend(['--smtp', smtp_host]) container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond + cmd = Command(pres, odoo_cmd, []) + docker_run(cmd.build(), logfile, args.build_dir, container_name, exposed_ports=[args.odoo_port, args.odoo_port + 1], cpu_limit=300) - # FIXME - # docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name, exposed_ports=[args.odoo_port, args.odoo_port + 1], cpu_limit=300) if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(name)s: %(message)s') parser = argparse.ArgumentParser() - subparser = parser.add_subparsers(help='commands') + subparser = parser.add_subparsers(dest='command', required='True', help='commands') p_build = subparser.add_parser('build', help='Build docker image') p_build.add_argument('build_dir') p_build.set_defaults(func=build) @@ -246,10 +272,13 @@ if __name__ == '__main__': p_test.add_argument('build_dir') p_test.add_argument('odoo_port', type=int) p_test.add_argument('db_name') - p_test.add_argument('--coverage', action='store_true', help= 'test a build with coverage') + group = p_test.add_mutually_exclusive_group() + group.add_argument('--coverage', action='store_true', help= 'test a build with coverage') + group.add_argument('--flamegraph', action='store_true', help= 'test a build and draw a flamegraph') p_test.add_argument('-i', dest='odoo_modules', default='web', help='Comma separated list of modules') p_test.add_argument('--kill', action='store_true', default=False, help='Also test container kill') p_test.add_argument('--dump', action='store_true', default=False, help='Test database export with pg_dump') p_test.add_argument('--run', action='store_true', default=False, help='Also test running (Warning: the container survives exit)') + p_test.add_argument('--env', action='store_true', default=False, help='Test passing environment variables') args = parser.parse_args() args.func(args) diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index 49de202d..5640c7a5 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -353,8 +353,7 @@ class ConfigStep(models.Model): zip_path = '/data/build/logs/%s.zip' % db_name cmd.finals.append(['pg_dump', db_name, '>', sql_dest]) cmd.finals.append(['cp', '-r', filestore_path, filestore_dest]) - cmd.finals.append(['cd', dump_dir]) - cmd.finals.append(['zip', '-rqm9', zip_path, '*']) + cmd.finals.append(['cd', dump_dir, '&&', 'zip', '-rmq9', zip_path, '*']) infos = '{\n "db_name": "%s",\n "build_id": %s,\n "shas": [%s]\n}' % (db_name, build.id, ', '.join(['"%s"' % commit for commit in build._get_all_commit()])) build.write_file('logs/%s/info.json' % db_name, infos)