[FIX] runbot: force repo update on runbot builders

When a runbot execute the cron_fetch_and_build method, the repo is
updated only if the webhook time is newer than the last fetch
time.

As the cron is now split into long running crons, the hook_time field is
cached. The runbot instance that sees a new build pending use this
cached value to estimate if the repo update is needed.

With this commit, the repo update is done right before exporting the
repo and only if the commit hash is not found.

As a bonus, the environment is reset in the long running cron of the
runbot builders to update the cached values.

runbot/models/build.py

@@ -510,6 +510,10 @@ class runbot_build(models.Model):
             os.makedirs(build._path("logs"), exist_ok=True)
             os.makedirs(build._server('addons'), exist_ok=True)
 
+            # update repo if needed
+            if not build.repo_id._hash_exists(build.name):
+                build.repo_id._update(build.repo_id)
+
             # checkout branch
             build.branch_id.repo_id._git_export(build.name, build._path())
 

runbot/models/repo.py

@@ -90,6 +90,15 @@ class runbot_repo(models.Model):
         p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
         p2.communicate()[0]
 
+    def _hash_exists(self, commit_hash):
+        """ Verify that a commit hash exists in the repo """
+        self.ensure_one()
+        try:
+            self._git(['cat-file', '-e', commit_hash])
+        except subprocess.CalledProcessError:
+            return False
+        return True
+
     def _github(self, url, payload=None, ignore_errors=False):
         """Return a http request to be sent to github"""
         for repo in self:
@@ -219,7 +228,7 @@ class runbot_repo(models.Model):
             _logger.info("Cloning repository '%s' in '%s'" % (repo.name, repo.path))
             subprocess.call(['git', 'clone', '--bare', repo.name, repo.path])
 
-    def _update_git(self):
+    def _update_git(self, force):
         """ Update the git repo on FS """
         self.ensure_one()
         repo = self
@@ -231,7 +240,7 @@ class runbot_repo(models.Model):
 
         # check for mode == hook
         fname_fetch_head = os.path.join(repo.path, 'FETCH_HEAD')
-        if os.path.isfile(fname_fetch_head):
+        if not force and os.path.isfile(fname_fetch_head):
             fetch_time = os.path.getmtime(fname_fetch_head)
             if repo.mode == 'hook' and repo.hook_time and dt2time(repo.hook_time) < fetch_time:
                 t0 = time.time()
@@ -241,11 +250,11 @@ class runbot_repo(models.Model):
 
         repo._git(['fetch', '-p', 'origin', '+refs/heads/*:refs/heads/*', '+refs/pull/*/head:refs/pull/*'])
 
-    def _update(self, repos):
+    def _update(self, repos, force=True):
         """ Update the physical git reposotories on FS"""
         for repo in repos:
             try:
-                repo._update_git()
+                repo._update_git(force)
             except Exception:
                 _logger.exception('Fail to update repo %s', repo.name)
 
@@ -371,7 +380,7 @@ class runbot_repo(models.Model):
         update_frequency = int(icp.get_param('runbot.runbot_update_frequency', default=10))
         while time.time() - start_time < timeout:
             repos = self.search([('mode', '!=', 'disabled')])
-            self._update(repos)
+            self._update(repos, force=False)
             self._create_pending_builds(repos)
             self.env.cr.commit()
             time.sleep(update_frequency)
@@ -388,8 +397,9 @@ class runbot_repo(models.Model):
         update_frequency = int(icp.get_param('runbot.runbot_update_frequency', default=10))
         while time.time() - start_time < timeout:
             repos = self.search([('mode', '!=', 'disabled')])
-            self._update(repos)
             self._scheduler(repos.ids)
             self.env.cr.commit()
+            self.env.reset()
+            self = self.env()[self._name]
             self._reload_nginx()
             time.sleep(update_frequency)

runbot/tests/test_cron.py

@@ -42,21 +42,19 @@ class Test_Cron(common.TransactionCase):
         self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_update_frequency', 1)
         ret = self.Repo._cron_fetch_and_schedule('runbotx.foo.com')
         self.assertEqual(None, ret)
-        mock_update.assert_called_with(self.Repo)
+        mock_update.assert_called_with(self.Repo, force=False)
         mock_create.assert_called_with(self.Repo)
 
     @patch('odoo.addons.runbot.models.repo.runbot_repo._get_cron_period')
     @patch('odoo.addons.runbot.models.repo.runbot_repo._reload_nginx')
     @patch('odoo.addons.runbot.models.repo.runbot_repo._scheduler')
-    @patch('odoo.addons.runbot.models.repo.runbot_repo._update')
     @patch('odoo.addons.runbot.models.repo.fqdn')
-    def test_cron_build(self, mock_fqdn, mock_update, mock_scheduler, mock_reload, mock_cron_period):
+    def test_cron_build(self, mock_fqdn, mock_scheduler, mock_reload, mock_cron_period):
         """ test that cron_fetch_and_build do its work """
         mock_fqdn.return_value = 'runbotx.foo.com'
         mock_cron_period.return_value = 2
         self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_update_frequency', 1)
         ret = self.Repo._cron_fetch_and_build('runbotx.foo.com')
         self.assertEqual(None, ret)
-        mock_update.assert_called_with(self.Repo)
         mock_scheduler.assert_called_with([])
         self.assertTrue(mock_reload.called)