From 820adae5ac94cb1f8818d6b022c6c7fbaab4709d Mon Sep 17 00:00:00 2001 From: Christophe Monniez Date: Tue, 25 Feb 2020 16:23:24 +0100 Subject: [PATCH] [IMP] runbot: auto disable host when git fetch fail From times to times, a git repo gets corrupted, making builds fail in chain. With this commit, the host on which the git fetch fails will be reserved if not more than half the hosts are reserved. --- runbot/models/host.py | 8 ++++++++ runbot/models/repo.py | 8 +++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/runbot/models/host.py b/runbot/models/host.py index d89d26db..347a80f9 100644 --- a/runbot/models/host.py +++ b/runbot/models/host.py @@ -103,3 +103,11 @@ class RunboHost(models.Model): def _total_workers(self): return sum(host.get_nb_worker() for host in self) + + def disable(self): + """ Reserve host if possible """ + self.ensure_one() + nb_hosts = self.env['runbot.host'].search_count([]) + nb_reserved = self.env['runbot.host'].search_count([('assigned_only', '=', True)]) + if nb_reserved < (nb_hosts / 2): + self.assigned_only = True diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 3817da82..ef4bb1b1 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -425,7 +425,13 @@ class runbot_repo(models.Model): # Extracted from update_git to be easily overriden in external module self.ensure_one() repo = self - repo._git(['fetch', '-p', 'origin', '+refs/heads/*:refs/heads/*', '+refs/pull/*/head:refs/pull/*']) + try: + repo._git(['fetch', '-p', 'origin', '+refs/heads/*:refs/heads/*', '+refs/pull/*/head:refs/pull/*']) + except subprocess.CalledProcessError as e: + message = 'Failed to fetch repo %s with return code %s. Original command was %s' % (repo.name, e.returncode, e.cmd) + _logger.exception(message) + host = self.env['runbot.host'].search([('name', '=', fqdn())]) + host.disable() def _update(self, force=True): """ Update the physical git reposotories on FS"""