diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 24f6bb5c..5498f940 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -374,13 +374,17 @@ class Repo(models.Model): def _source_path(self, *path_parts): return self.env['runbot.runbot']._path('sources', sanitize(self.name), *path_parts) - def _git(self, cmd, errors='strict'): + def _get_git_command(self, cmd, errors='strict'): """Execute a git command 'cmd'""" self.ensure_one() config_args = [] if self.identity_file: config_args = ['-c', 'core.sshCommand=ssh -i %s/.ssh/%s' % (str(Path.home()), self.identity_file)] cmd = ['git', '-C', self.path] + config_args + cmd + return cmd + + def _git(self, cmd, errors='strict'): + cmd = self._get_git_command(cmd, errors) _logger.info("git command: %s", ' '.join(cmd)) return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode(errors=errors) diff --git a/runbot/models/runbot.py b/runbot/models/runbot.py index b4c345d5..ed17fdf4 100644 --- a/runbot/models/runbot.py +++ b/runbot/models/runbot.py @@ -383,16 +383,6 @@ class Runbot(models.AbstractModel): if ignored: _logger.info('docker (%s) not deleted because not dest format', list(ignored)) - def _git_gc(self, host): - """ - cleanup and optimize git repositories on the host - """ - for repo in self.env['runbot.repo'].search([]): - try: - repo._git(['gc', '--prune=all', '--quiet']) - except CalledProcessError as e: - message = f'git gc failed for {repo.name} on {host.name} with exit status {e.returncode} and message "{e.output[:60]} ..."' - self._warning(message) def _warning(self, message, *args): if args: diff --git a/runbot_builder/builder.py b/runbot_builder/builder.py index 4ddd3ea7..9d6697bf 100755 --- a/runbot_builder/builder.py +++ b/runbot_builder/builder.py @@ -28,7 +28,9 @@ class BuilderClient(RunbotClient): self.host._set_psql_conn_count() self.host._docker_build() self.env['runbot.repo']._update_git_config() + self.env.cr.commit() self.git_gc() + self.env.cr.commit() return self.env['runbot.runbot']._scheduler_loop_turn(self.host) diff --git a/runbot_builder/leader.py b/runbot_builder/leader.py index 91ece899..8e785759 100755 --- a/runbot_builder/leader.py +++ b/runbot_builder/leader.py @@ -20,7 +20,9 @@ class LeaderClient(RunbotClient): # Conductor, Director, Main, Maestro, Lead def loop_turn(self): if self.count == 0: self.env['runbot.repo']._update_git_config() + self.env.cr.commit() self.git_gc() + self.env.cr.commit() return self.env['runbot.runbot']._fetch_loop_turn(self.host, self.pull_info_failures) diff --git a/runbot_builder/tools.py b/runbot_builder/tools.py index 0d9a023c..c5f4820c 100644 --- a/runbot_builder/tools.py +++ b/runbot_builder/tools.py @@ -9,6 +9,7 @@ import sys import threading import time import signal +import subprocess from contextlib import nullcontext from datetime import datetime, timedelta, timezone @@ -111,7 +112,25 @@ class RunbotClient(): """ git gc once a day """ if self.next_git_gc_date < datetime.now(): _logger.info('Starting git gc on repositories') - self.env['runbot.runbot']._git_gc(self.host) + commands = [] + host_name = self.host.name + for repo in self.env['runbot.repo'].search([]): + commands.append(repo.name, (repo._get_git_command(['gc', '--prune=all', '--quiet']))) + self.env.cr.rollback() + # gc commands can be slow, rollbacking to avoid to keep a transaction idle for multiple minutes. + messages = [] + for repo_name, command in commands: + try: + start = time.time() + subprocess.check_output(command, stderr=subprocess.STDOUT).decode() + _logger.info('Git gc on %s took %ss', repo_name, time.time() - start) + except subprocess.CalledProcessError as e: + message = f'git gc failed for {repo_name} on {host_name} with exit status {e.returncode} and message "{e.output[:60]} ..."' + messages.append(message) + for message in messages: + self.env['runbot.runbot']._warning(message) + + self.env.cr.commit() self.update_next_git_gc_date() def run(client_class):