diff --git a/runbot/container.py b/runbot/container.py index 22ea097a..bc27bfea 100644 --- a/runbot/container.py +++ b/runbot/container.py @@ -83,6 +83,7 @@ def docker_run(run_cmd, log_path, build_dir, container_name, exposed_ports=None, :param container_name: used to give a name to the container for later reference :param exposed_ports: if not None, starting at 8069, ports will be exposed as exposed_ports numbers :params ro_volumes: dict of dest:source volumes to mount readonly in builddir + :params env_variables: list of environment variables """ _logger.debug('Docker run command: %s', run_cmd) logs = open(log_path, 'w') @@ -159,6 +160,7 @@ def build(args): logdir = os.path.join(args.build_dir, 'logs') os.makedirs(logdir, exist_ok=True) logfile = os.path.join(logdir, 'logs-build.txt') + _logger.info('Logfile is in %s', logfile) docker_build(logfile, args.build_dir) _logger.info('Finished building the base image container') @@ -176,14 +178,54 @@ def tests(args): logfile = os.path.join(args.build_dir, 'logs', 'logs-build.txt') docker_build(logfile, args.build_dir) + with open(os.path.join(args.build_dir, 'odoo-bin'), 'r') as exfile: + py_version = '3' if 'python3' in exfile.readline() else '' + + # Test environment variables + if args.env: + cmd = Command(None, ['echo testa is $TESTA and testb is $TESTB '], None) + env_variables = ['TESTA=test a', 'TESTB="test b"'] + env_log = os.path.join(args.build_dir, 'logs', 'logs-env.txt') + container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond + docker_run(cmd.build(), env_log, args.build_dir, container_name, env_variables=env_variables) + expected = 'testa is test a and testb is "test b"' + time.sleep(3) # ugly sleep to wait for docker process to flush the log file + assert expected in open(env_log,'r').read() + # Test testing - odoo_cmd = ['/data/build/odoo-bin', '-d %s' % args.db_name, '--addons-path=/data/build/addons', '--data-dir', '/data/build/datadir', '-r %s' % os.getlogin(), '-i', args.odoo_modules, '--test-enable', '--stop-after-init', '--max-cron-threads=0'] + pres = [['sudo', 'pip%s' % py_version, 'install', '-r', '/data/build/requirements.txt']] + posts = None + python_params = [] + if args.coverage: + omit = ['--omit', '*__manifest__.py'] + python_params = [ '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + omit + posts = [['python%s' % py_version, "-m", "coverage", "html", "-d", "/data/build/coverage", "--ignore-errors"]] + os.makedirs(os.path.join(args.build_dir, 'coverage'), exist_ok=True) + elif args.flamegraph: + flame_log = '/data/build/logs/flame.log' + python_params = ['-m', 'flamegraph', '-o', flame_log] + odoo_cmd = ['python%s' % py_version ] + python_params + ['/data/build/odoo-bin', '-d %s' % args.db_name, '--addons-path=/data/build/addons', '--data-dir', '/data/build/datadir', '-r %s' % os.getlogin(), '-i', args.odoo_modules, '--test-enable', '--stop-after-init', '--max-cron-threads=0'] + cmd = Command(pres, odoo_cmd, posts) + + if args.dump: + os.makedirs(os.path.join(args.build_dir, 'logs', args.db_name), exist_ok=True) + dump_dir = '/data/build/logs/%s/' % args.db_name + sql_dest = '%s/dump.sql' % dump_dir + filestore_path = '/data/build/datadir/filestore/%s' % args.db_name + filestore_dest = '%s/filestore/' % dump_dir + zip_path = '/data/build/logs/%s.zip' % args.db_name + cmd.finals.append(['pg_dump', args.db_name, '>', sql_dest]) + cmd.finals.append(['cp', '-r', filestore_path, filestore_dest]) + cmd.finals.append(['cd', dump_dir, '&&', 'zip', '-rm9', zip_path, '*']) + + if args.flamegraph: + cmd.finals.append(['flamegraph.pl', '--title', 'Flamegraph', flame_log, '>', '/data/build/logs/flame.svg']) + cmd.finals.append(['gzip', '-f', flame_log]) if args.kill: logfile = os.path.join(args.build_dir, 'logs', 'logs-partial.txt') container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond - # FIXME - # docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name) + docker_run(cmd.build(), logfile, args.build_dir, container_name) # Test stopping the container _logger.info('Waiting 30 sec before killing the build') time.sleep(30) @@ -193,34 +235,18 @@ def tests(args): # Test full testing logfile = os.path.join(args.build_dir, 'logs', 'logs-full-test.txt') container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond - if args.coverage: - omit = ['--omit', '*__manifest__.py'] - with open(os.path.join(args.build_dir, 'odoo-bin'), 'r') as exfile: - pyversion = 'python3' if 'python3' in exfile.readline() else 'python' - odoo_cmd = [ pyversion, '-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + omit + odoo_cmd - # FIXME - # docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name) + docker_run(cmd.build(), logfile, args.build_dir, container_name) time.sleep(1) # give time for the container to start while docker_is_running(container_name): time.sleep(10) _logger.info("Waiting for %s to stop", container_name) - if args.dump: - _logger.info("Testing pg_dump") - logfile = os.path.join(args.build_dir, 'logs', 'logs-pg_dump.txt') - container_name = 'odoo-container-test-pg_dump-%s' % datetime.datetime.now().microsecond - docker_pg_dump_cmd = 'cd /data/build/datadir && pg_dump -U %s -f db_export.sql %s' % (os.getlogin(), args.db_name) - docker_run(docker_pg_dump_cmd, logfile, args.build_dir, container_name) - time.sleep(1) - while docker_is_running(container_name): - time.sleep(10) - _logger.info("Waiting for %s to stop", container_name) - if args.run: # Test running logfile = os.path.join(args.build_dir, 'logs', 'logs-running.txt') odoo_cmd = [ + 'python%s' % py_version, '/data/build/odoo-bin', '-d %s' % args.db_name, '--db-filter', '%s.*$' % args.db_name, '--addons-path=/data/build/addons', '-r %s' % os.getlogin(), '-i', 'web', '--max-cron-threads=1', @@ -230,14 +256,14 @@ def tests(args): if smtp_host: odoo_cmd.extend(['--smtp', smtp_host]) container_name = 'odoo-container-test-%s' % datetime.datetime.now().microsecond + cmd = Command(pres, odoo_cmd, []) + docker_run(cmd.build(), logfile, args.build_dir, container_name, exposed_ports=[args.odoo_port, args.odoo_port + 1], cpu_limit=300) - # FIXME - # docker_run(build_odoo_cmd(odoo_cmd), logfile, args.build_dir, container_name, exposed_ports=[args.odoo_port, args.odoo_port + 1], cpu_limit=300) if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(name)s: %(message)s') parser = argparse.ArgumentParser() - subparser = parser.add_subparsers(help='commands') + subparser = parser.add_subparsers(dest='command', required='True', help='commands') p_build = subparser.add_parser('build', help='Build docker image') p_build.add_argument('build_dir') p_build.set_defaults(func=build) @@ -246,10 +272,13 @@ if __name__ == '__main__': p_test.add_argument('build_dir') p_test.add_argument('odoo_port', type=int) p_test.add_argument('db_name') - p_test.add_argument('--coverage', action='store_true', help= 'test a build with coverage') + group = p_test.add_mutually_exclusive_group() + group.add_argument('--coverage', action='store_true', help= 'test a build with coverage') + group.add_argument('--flamegraph', action='store_true', help= 'test a build and draw a flamegraph') p_test.add_argument('-i', dest='odoo_modules', default='web', help='Comma separated list of modules') p_test.add_argument('--kill', action='store_true', default=False, help='Also test container kill') p_test.add_argument('--dump', action='store_true', default=False, help='Test database export with pg_dump') p_test.add_argument('--run', action='store_true', default=False, help='Also test running (Warning: the container survives exit)') + p_test.add_argument('--env', action='store_true', default=False, help='Test passing environment variables') args = parser.parse_args() args.func(args) diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index 49de202d..5640c7a5 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -353,8 +353,7 @@ class ConfigStep(models.Model): zip_path = '/data/build/logs/%s.zip' % db_name cmd.finals.append(['pg_dump', db_name, '>', sql_dest]) cmd.finals.append(['cp', '-r', filestore_path, filestore_dest]) - cmd.finals.append(['cd', dump_dir]) - cmd.finals.append(['zip', '-rqm9', zip_path, '*']) + cmd.finals.append(['cd', dump_dir, '&&', 'zip', '-rmq9', zip_path, '*']) infos = '{\n "db_name": "%s",\n "build_id": %s,\n "shas": [%s]\n}' % (db_name, build.id, ', '.join(['"%s"' % commit for commit in build._get_all_commit()])) build.write_file('logs/%s/info.json' % db_name, infos)