[IMP] runbot: build docker image once per loop turn

At this moment, the Docker image is built at the beginning of each
runbot build. This blocks the _scheduler while the image is built.

With this commit, the image is built before calling the _scheduler and
is not linked to a runbot build.

Also, the necessary dirs are created in the static path before starting
the loop.

.gitignore

@@ -10,3 +10,4 @@ runbot/static/repo
 runbot/static/sources
 runbot/static/nginx
 runbot/static/databases
+runbot/static/docker

runbot/models/build.py

@@ -602,8 +602,6 @@ class runbot_build(models.Model):
                 # notify pending build - avoid confusing users by saying nothing
                 build._github_status()
                 os.makedirs(build._path('logs'), exist_ok=True)
-                build._log('_schedule', 'Building docker image')
-                docker_build(build._path('logs', 'docker_build.txt'), build._path())
             except Exception:
                 _logger.exception('Failed initiating build %s', build.dest)
                 build._log('_schedule', 'Failed initiating build')

runbot/models/host.py

@@ -1,6 +1,9 @@
 import logging
+import os
+
 from odoo import models, fields, api
 from ..common import fqdn, local_pgadmin_cursor
+from ..container import docker_build
 _logger = logging.getLogger(__name__)
 
 
@@ -44,6 +47,23 @@ class RunboHost(models.Model):
             values['disp_name'] = values['name']
         return super().create(values)
 
+    def _bootstrap(self):
+        """ Create needed directories in static """
+        dirs = ['build', 'nginx', 'repo', 'sources', 'src', 'docker']
+        static_path = self._get_work_path()
+        static_dirs = {d: os.path.join(static_path, d) for d in dirs}
+        for dir, path in static_dirs.items():
+            os.makedirs(path, exist_ok=True)
+
+    def _docker_build(self):
+        """ build docker image """
+        static_path = self._get_work_path()
+        log_path = os.path.join(static_path, 'docker', 'docker_build.txt')
+        docker_build(log_path, static_path)
+
+    def _get_work_path(self):
+        return os.path.abspath(os.path.join(os.path.dirname(__file__), '../static'))
+
     @api.model
     def _get_current(self):
         name = fqdn()

runbot/models/repo.py

@@ -660,6 +660,7 @@ class runbot_repo(models.Model):
 
         host = self.env['runbot.host']._get_current()
         host.set_psql_conn_count()
+        host._bootstrap()
         host.last_start_loop = fields.Datetime.now()
         
         self._commit()
@@ -673,6 +674,7 @@ class runbot_repo(models.Model):
         self.env['runbot.build']._local_cleanup()
         # 3. docker cleanup
         self.env['runbot.repo']._docker_cleanup()
+        host._docker_build()
 
         timeout = self._get_cron_period()
         icp = self.env['ir.config_parameter']

runbot/tests/test_cron.py

@@ -44,9 +44,11 @@ class Test_Cron(RunbotCase):
         mock_update.assert_called_with(force=False)
         mock_create.assert_called_with()
 
+    @patch('odoo.addons.runbot.models.host.RunboHost._docker_build')
+    @patch('odoo.addons.runbot.models.host.RunboHost._bootstrap')
     @patch('odoo.addons.runbot.models.repo.runbot_repo._reload_nginx')
     @patch('odoo.addons.runbot.models.repo.runbot_repo._scheduler')
-    def test_cron_build(self, mock_scheduler, mock_reload):
+    def test_cron_build(self, mock_scheduler, mock_reload, mock_host_bootstrap, mock_host_docker_build):
         """ test that cron_fetch_and_build do its work """
         hostname = 'host.runbot.com'
         self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_update_frequency', 1)
@@ -56,6 +58,8 @@ class Test_Cron(RunbotCase):
         ret = self.Repo._cron_fetch_and_build(hostname)
         self.assertEqual(None, ret)
         mock_scheduler.assert_called()
+        mock_host_bootstrap.assert_called()
+        mock_host_docker_build.assert_called()
         host = self.env['runbot.host'].search([('name', '=', hostname)])
         self.assertEqual(host.name, hostname, 'A new host should have been created')
         self.assertGreater(host.psql_conn_count, 0, 'A least one connection should exist on the current psql instance')

runbot_builder/builder.py

@@ -27,6 +27,7 @@ class RunbotClient():
         signal.signal(signal.SIGINT, self.signal_handler)
         signal.signal(signal.SIGTERM, self.signal_handler)
         host = self.env['runbot.host']._get_current()
+        host._bootstrap()
         count = 0
         while True:
             try:
@@ -38,6 +39,7 @@ class RunbotClient():
                     self.env['runbot.build']._local_cleanup()
                     self.env['runbot.repo']._docker_cleanup()
                     host.set_psql_conn_count()
+                    host._docker_build()
                     _logger.info('Scheduling...')
                 count += 1
                 sleep_time = self.env['runbot.repo']._scheduler_loop_turn(host)