From 0c609ac533e37413a70332d67bdf7ff0c1d4294a Mon Sep 17 00:00:00 2001 From: Xavier-Do Date: Fri, 30 Jun 2023 15:16:33 +0200 Subject: [PATCH] [IMP] runbot: improve cleanup perfs The current version will read ~100000 build This one will avoid that by checking the date of the gcstamp instead --- runbot/models/build.py | 59 +++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/runbot/models/build.py b/runbot/models/build.py index f86fb2ba..54292cdc 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -517,7 +517,6 @@ class BuildResult(models.Model): ignored = set() icp = self.env['ir.config_parameter'] hide_in_logs = icp.get_param('runbot.runbot_db_template', default='template0') - full_gc_days = int(icp.get_param('runbot.full_gc_days', default=365)) for dest in dest_list: build = self._build_from_dest(dest) @@ -536,9 +535,8 @@ class BuildResult(models.Model): for build in existing: if build.gc_date < fields.datetime.now(): if build.local_state == 'done': - full = build.gc_date + datetime.timedelta(days=(full_gc_days)) < fields.datetime.now() for db in dest_by_builds_ids[build.id]: - yield (db, full) + yield db elif build.local_state != 'running': _logger.warning('db (%s) not deleted because state is not done', " ".join(dest_by_builds_ids[build.id])) @@ -556,7 +554,7 @@ class BuildResult(models.Model): for dest in dest_list: build = self._build_from_dest(dest) if build and build in self: - yield (dest, full) + yield dest elif not build: _logger.info('%s (%s) skipped because not dest format', label, dest) _filter = filter_ids @@ -566,36 +564,45 @@ class BuildResult(models.Model): log_db = self.env['ir.config_parameter'].get_param('runbot.logdb_name') existing_db = [db for db in list_local_dbs(additionnal_conditions=additionnal_conditions) if db != log_db] - for db, _ in _filter(dest_list=existing_db, label='db'): + for db in _filter(dest_list=existing_db, label='db'): self._logger('Removing database') self._local_pg_dropdb(db) builds_dir = Path(self.env['runbot.runbot']._root()) / 'build' - - if force is True: - dests = [(build.dest, full) for build in self] + if force: + dest_list = [build.dest for build in self] else: - dests = _filter(dest_list=(p.name for p in builds_dir.iterdir()), label='workspace') + dest_list = (p.name for p in builds_dir.iterdir()) - for dest, full in dests: + icp = self.env['ir.config_parameter'] + full_gc_days = int(icp.get_param('runbot.full_gc_days', default=365)) + full_gc_secondes = full_gc_days * 24 * 60 * 60 + now = time.time() + + candidate_for_partial_gc = [] + for dest in dest_list: build_dir = Path(builds_dir) / dest - if full: - _logger.info('Removing build dir "%s"', dest) - shutil.rmtree(build_dir, ignore_errors=True) - continue gcstamp = build_dir / '.gcstamp' - if gcstamp.exists(): - continue - for bdir_file in build_dir.iterdir(): - if bdir_file.is_dir() and bdir_file.name not in ('logs', 'tests'): - shutil.rmtree(bdir_file) - elif bdir_file.name == 'logs': - for log_file_path in bdir_file.iterdir(): - if log_file_path.is_dir(): - shutil.rmtree(log_file_path) - elif log_file_path.name in ('run.txt', 'wake_up.txt') or not log_file_path.name.endswith('.txt'): - log_file_path.unlink() - gcstamp.write_text(f'gc date: {datetime.datetime.now()}') + try: + if (force and full) or gcstamp.stat().st_ctime + full_gc_secondes < now: + _logger.info('Removing build dir "%s"', dest) + shutil.rmtree(build_dir, ignore_errors=True) + continue + except(FileNotFoundError): + candidate_for_partial_gc.append(dest) + if candidate_for_partial_gc: + for dest in _filter(candidate_for_partial_gc, label='workspace'): + build_dir = Path(builds_dir) / dest + for bdir_file in build_dir.iterdir(): + if bdir_file.is_dir() and bdir_file.name not in ('logs', 'tests'): + shutil.rmtree(bdir_file) + elif bdir_file.name == 'logs': + for log_file_path in bdir_file.iterdir(): + if log_file_path.is_dir(): + shutil.rmtree(log_file_path) + elif log_file_path.name in ('run.txt', 'wake_up.txt') or not log_file_path.name.endswith('.txt'): + log_file_path.unlink() + gcstamp.write_text(f'gc date: {datetime.datetime.now()}') def _find_port(self): # currently used port