NextERP/runbot
5
0
Fork 0
mirror of https://github.com/odoo/runbot.git synced 2025-03-15 23:45:44 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

[IMP] runbot: improve cleanup perfs

Browse Source 
The current version will read ~100000 build

This one will avoid that by checking the date of the gcstamp instead
This commit is contained in:
Xavier-Do 2023-06-30 15:16:33 +02:00
parent ab194610b0
commit 423b34433d
1 changed files with 33 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
runbot/models/build.py
View File

@ -509,7 +509,6 @@ class BuildResult(models.Model):
ignored = set()
icp = self.env['ir.config_parameter']
hide_in_logs = icp.get_param('runbot.runbot_db_template', default='template0')
full_gc_days = int(icp.get_param('runbot.full_gc_days', default=365))
for dest in dest_list:
build = self._build_from_dest(dest)
@ -528,9 +527,8 @@ class BuildResult(models.Model):
for build in existing:
if build.gc_date < fields.datetime.now():
if build.local_state == 'done':
full = build.gc_date + datetime.timedelta(days=(full_gc_days)) < fields.datetime.now()
for db in dest_by_builds_ids[build.id]:
yield (db, full)
yield db
elif build.local_state != 'running':
_logger.warning('db (%s) not deleted because state is not done', " ".join(dest_by_builds_ids[build.id]))
@ -548,7 +546,7 @@ class BuildResult(models.Model):
for dest in dest_list:
build = self._build_from_dest(dest)
if build and build in self:
yield (dest, full)
yield dest
elif not build:
_logger.info('%s (%s) skipped because not dest format', label, dest)
_filter = filter_ids
@ -558,36 +556,45 @@ class BuildResult(models.Model):
log_db = self.env['ir.config_parameter'].get_param('runbot.logdb_name')
existing_db = [db for db in list_local_dbs(additionnal_conditions=additionnal_conditions) if db != log_db]
for db, _ in _filter(dest_list=existing_db, label='db'):
for db in _filter(dest_list=existing_db, label='db'):
self._logger('Removing database')
self._local_pg_dropdb(db)
builds_dir = Path(self.env['runbot.runbot']._root()) / 'build'
if force is True:
dests = [(build.dest, full) for build in self]
if force:
dest_list = [build.dest for build in self]
else:
dests = _filter(dest_list=(p.name for p in builds_dir.iterdir()), label='workspace')
dest_list = (p.name for p in builds_dir.iterdir())
for dest, full in dests:
icp = self.env['ir.config_parameter']
full_gc_days = int(icp.get_param('runbot.full_gc_days', default=365))
full_gc_secondes = full_gc_days * 24 * 60 * 60
now = time.time()
candidate_for_partial_gc = []
for dest in dest_list:
build_dir = Path(builds_dir) / dest
if full:
_logger.info('Removing build dir "%s"', dest)
shutil.rmtree(build_dir, ignore_errors=True)
continue
gcstamp = build_dir / '.gcstamp'
if gcstamp.exists():
continue
for bdir_file in build_dir.iterdir():
if bdir_file.is_dir() and bdir_file.name not in ('logs', 'tests'):
shutil.rmtree(bdir_file)
elif bdir_file.name == 'logs':
for log_file_path in bdir_file.iterdir():
if log_file_path.is_dir():
shutil.rmtree(log_file_path)
elif log_file_path.name in ('run.txt', 'wake_up.txt') or not log_file_path.name.endswith('.txt'):
log_file_path.unlink()
gcstamp.write_text(f'gc date: {datetime.datetime.now()}')
try:
if (force and full) or gcstamp.stat().st_ctime + full_gc_secondes < now:
_logger.info('Removing build dir "%s"', dest)
shutil.rmtree(build_dir, ignore_errors=True)
continue
except(FileNotFoundError):
candidate_for_partial_gc.append(dest)
if candidate_for_partial_gc:
for dest in _filter(candidate_for_partial_gc, label='workspace'):
build_dir = Path(builds_dir) / dest
for bdir_file in build_dir.iterdir():
if bdir_file.is_dir() and bdir_file.name not in ('logs', 'tests'):
shutil.rmtree(bdir_file)
elif bdir_file.name == 'logs':
for log_file_path in bdir_file.iterdir():
if log_file_path.is_dir():
shutil.rmtree(log_file_path)
elif log_file_path.name in ('run.txt', 'wake_up.txt') or not log_file_path.name.endswith('.txt'):
log_file_path.unlink()
gcstamp.write_text(f'gc date: {datetime.datetime.now()}')
def _find_port(self):
# currently used port