From 1617a2e339bb94f8c110dd5ccecb43965368e2e3 Mon Sep 17 00:00:00 2001 From: Christophe Monniez Date: Mon, 25 Feb 2019 15:18:10 +0100 Subject: [PATCH] [FIX] runbot: force repo update on runbot builders When a runbot execute the cron_fetch_and_build method, the repo is updated only if the webhook time is newer than the last fetch time. As the cron is now split into long running crons, the hook_time field is cached. The runbot instance that sees a new build pending use this cached value to estimate if the repo update is needed. With this commit, the repo update is done right before exporting the repo and only if the commit hash is not found. As a bonus, the environment is reset in the long running cron of the runbot builders to update the cached values. --- runbot/models/build.py | 4 ++++ runbot/models/repo.py | 22 ++++++++++++++++------ runbot/tests/test_cron.py | 6 ++---- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/runbot/models/build.py b/runbot/models/build.py index 24911305..d304904f 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -510,6 +510,10 @@ class runbot_build(models.Model): os.makedirs(build._path("logs"), exist_ok=True) os.makedirs(build._server('addons'), exist_ok=True) + # update repo if needed + if not build.repo_id._hash_exists(build.name): + build.repo_id._update(build.repo_id) + # checkout branch build.branch_id.repo_id._git_export(build.name, build._path()) diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 600b31c4..d07570c4 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -90,6 +90,15 @@ class runbot_repo(models.Model): p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. p2.communicate()[0] + def _hash_exists(self, commit_hash): + """ Verify that a commit hash exists in the repo """ + self.ensure_one() + try: + self._git(['cat-file', '-e', commit_hash]) + except subprocess.CalledProcessError: + return False + return True + def _github(self, url, payload=None, ignore_errors=False): """Return a http request to be sent to github""" for repo in self: @@ -219,7 +228,7 @@ class runbot_repo(models.Model): _logger.info("Cloning repository '%s' in '%s'" % (repo.name, repo.path)) subprocess.call(['git', 'clone', '--bare', repo.name, repo.path]) - def _update_git(self): + def _update_git(self, force): """ Update the git repo on FS """ self.ensure_one() repo = self @@ -231,7 +240,7 @@ class runbot_repo(models.Model): # check for mode == hook fname_fetch_head = os.path.join(repo.path, 'FETCH_HEAD') - if os.path.isfile(fname_fetch_head): + if not force and os.path.isfile(fname_fetch_head): fetch_time = os.path.getmtime(fname_fetch_head) if repo.mode == 'hook' and repo.hook_time and dt2time(repo.hook_time) < fetch_time: t0 = time.time() @@ -241,11 +250,11 @@ class runbot_repo(models.Model): repo._git(['fetch', '-p', 'origin', '+refs/heads/*:refs/heads/*', '+refs/pull/*/head:refs/pull/*']) - def _update(self, repos): + def _update(self, repos, force=True): """ Update the physical git reposotories on FS""" for repo in repos: try: - repo._update_git() + repo._update_git(force) except Exception: _logger.exception('Fail to update repo %s', repo.name) @@ -371,7 +380,7 @@ class runbot_repo(models.Model): update_frequency = int(icp.get_param('runbot.runbot_update_frequency', default=10)) while time.time() - start_time < timeout: repos = self.search([('mode', '!=', 'disabled')]) - self._update(repos) + self._update(repos, force=False) self._create_pending_builds(repos) self.env.cr.commit() time.sleep(update_frequency) @@ -388,8 +397,9 @@ class runbot_repo(models.Model): update_frequency = int(icp.get_param('runbot.runbot_update_frequency', default=10)) while time.time() - start_time < timeout: repos = self.search([('mode', '!=', 'disabled')]) - self._update(repos) self._scheduler(repos.ids) self.env.cr.commit() + self.env.reset() + self = self.env()[self._name] self._reload_nginx() time.sleep(update_frequency) diff --git a/runbot/tests/test_cron.py b/runbot/tests/test_cron.py index dd30afa3..2c28bc49 100644 --- a/runbot/tests/test_cron.py +++ b/runbot/tests/test_cron.py @@ -42,21 +42,19 @@ class Test_Cron(common.TransactionCase): self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_update_frequency', 1) ret = self.Repo._cron_fetch_and_schedule('runbotx.foo.com') self.assertEqual(None, ret) - mock_update.assert_called_with(self.Repo) + mock_update.assert_called_with(self.Repo, force=False) mock_create.assert_called_with(self.Repo) @patch('odoo.addons.runbot.models.repo.runbot_repo._get_cron_period') @patch('odoo.addons.runbot.models.repo.runbot_repo._reload_nginx') @patch('odoo.addons.runbot.models.repo.runbot_repo._scheduler') - @patch('odoo.addons.runbot.models.repo.runbot_repo._update') @patch('odoo.addons.runbot.models.repo.fqdn') - def test_cron_build(self, mock_fqdn, mock_update, mock_scheduler, mock_reload, mock_cron_period): + def test_cron_build(self, mock_fqdn, mock_scheduler, mock_reload, mock_cron_period): """ test that cron_fetch_and_build do its work """ mock_fqdn.return_value = 'runbotx.foo.com' mock_cron_period.return_value = 2 self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_update_frequency', 1) ret = self.Repo._cron_fetch_and_build('runbotx.foo.com') self.assertEqual(None, ret) - mock_update.assert_called_with(self.Repo) mock_scheduler.assert_called_with([]) self.assertTrue(mock_reload.called)