NextERP/runbot
5
0
Fork 0
mirror of https://github.com/odoo/runbot.git synced 2025-03-27 13:25:47 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

[IMP] runbot: speedup build garbage collecting

Browse Source 
When the builds directory is filled with a lot of build directories
(around 100000) the garbage collection process may take up to 2 minutes.
The root cause is that each build directory is scanned to clean it up
even if it was already cleaned.

With this commit, a stamp file is used to mark directories that were
already garbage collected.
This commit is contained in:
Christophe Monniez 2022-09-20 08:41:19 +02:00 committed by xdo
parent 7642bffda3
commit 309aeaa32e
2 changed files with 17 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
runbot/models/build.py
View File

@ -17,8 +17,8 @@ from odoo.http import request
from odoo.tools import appdirs from odoo.tools import appdirs
from odoo.tools.safe_eval import safe_eval from odoo.tools.safe_eval import safe_eval
from collections import defaultdict from collections import defaultdict
from pathlib import Path
from psycopg2 import sql from psycopg2 import sql
from subprocess import CalledProcessError
import getpass import getpass
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
@ -535,32 +535,32 @@ class BuildResult(models.Model):
self._logger('Removing database') self._logger('Removing database')
self._local_pg_dropdb(db) self._local_pg_dropdb(db)
root = self.env['runbot.runbot']._root() builds_dir = Path(self.env['runbot.runbot']._root()) / 'build'
builds_dir = os.path.join(root, 'build')
if force is True: if force is True:
dests = [(build.dest, full) for build in self] dests = [(build.dest, full) for build in self]
else: else:
dests = _filter(dest_list=os.listdir(builds_dir), label='workspace') dests = _filter(dest_list=builds_dir.iterdir(), label='workspace')
for dest, full in dests: for dest, full in dests:
build_dir = os.path.join(builds_dir, dest) build_dir = Path(builds_dir) / dest
if full: if full:
_logger.info('Removing build dir "%s"', dest) _logger.info('Removing build dir "%s"', dest)
shutil.rmtree(build_dir, ignore_errors=True) shutil.rmtree(build_dir, ignore_errors=True)
continue continue
for f in os.listdir(build_dir): gcstamp = build_dir / '.gcstamp'
path = os.path.join(build_dir, f) if gcstamp.exists():
if os.path.isdir(path) and f not in ('logs', 'tests'): continue
shutil.rmtree(path) for bdir_file in build_dir.iterdir():
elif f == 'logs': if bdir_file.is_dir() and bdir_file.name not in ('logs', 'tests'):
log_path = os.path.join(build_dir, 'logs') shutil.rmtree(bdir_file)
for f in os.listdir(log_path): elif bdir_file.name == 'logs':
log_file_path = os.path.join(log_path, f) for log_file_path in (bdir_file / 'logs').iterdir():
if os.path.isdir(log_file_path): if log_file_path.is_dir():
shutil.rmtree(log_file_path) shutil.rmtree(log_file_path)
elif f in ('run.txt', 'wake_up.txt') or not f.endswith('.txt'): elif log_file_path.name in ('run.txt', 'wake_up.txt') or not log_file_path.name.endswith('.txt'):
os.unlink(log_file_path) log_file_path.unlink()
gcstamp.write_text(f'gc date: {datetime.datetime.now()}')
def _find_port(self): def _find_port(self):
# currently used port # currently used port

2
runbot/tests/test_build.py
View File

@ -298,7 +298,7 @@ class TestBuildResult(RunbotCase):
# test the real _local_cleanup method # test the real _local_cleanup method
self.stop_patcher('_local_cleanup_patcher') self.stop_patcher('_local_cleanup_patcher')
self.start_patcher('build_local_pgadmin_cursor_patcher', 'odoo.addons.runbot.models.build.local_pgadmin_cursor') self.start_patcher('build_local_pgadmin_cursor_patcher', 'odoo.addons.runbot.models.build.local_pgadmin_cursor')
self.start_patcher('build_os_listdirr_patcher', 'odoo.addons.runbot.models.build.os.listdir') self.start_patcher('build_path_patcher', 'odoo.addons.runbot.models.build.Path')
dbname = '%s-foobar' % build.dest dbname = '%s-foobar' % build.dest
self.start_patcher('list_local_dbs_patcher', 'odoo.addons.runbot.models.build.list_local_dbs', return_value=[dbname]) self.start_patcher('list_local_dbs_patcher', 'odoo.addons.runbot.models.build.list_local_dbs', return_value=[dbname])