NextERP/runbot
5
0
Fork 0
mirror of https://github.com/odoo/runbot.git synced 2025-03-15 23:45:44 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

[IMP] runbot: fully garbage collect old builds

Browse Source 
When a build is garbage collected and the local folders are cleaned, the
log files are kept to allow build errors investigations.

With this commit, a full local cleanup will be done 365 days after the
garbage collect.  This means that the build dir will be completely
removed.  The default number of days can be tweaked with the
`runbot.full_gc_days` ir.config_parameter.

Also, the _local_cleanup method can be called with a `full` boolean
parameter to force a full cleanup. e.g.: when called in a config step.

While at it, the res_config_settings view is a bit reorganized.
This commit is contained in:
Christophe Monniez 2022-02-07 14:50:26 +01:00 committed by xdo
parent 2f7c752793
commit a931af2fed
3 changed files with 39 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
runbot/models/build.py
View File

@ -393,7 +393,6 @@ class BuildResult(models.Model):
@api.depends_context('batch')
def _compute_build_url(self):
batch = self.env.context.get('batch')
print(self.env.context)
for build in self:
if batch:
build.build_url = "/runbot/batch/%s/build/%s" % (batch.id, build.id)
@ -483,6 +482,7 @@ class BuildResult(models.Model):
ignored = set()
icp = self.env['ir.config_parameter']
hide_in_logs = icp.get_param('runbot.runbot_db_template', default='template0')
full_gc_days = int(icp.get_param('runbot.full_gc_days', default=365))
for dest in dest_list:
build = self._build_from_dest(dest)
@ -499,14 +499,15 @@ class BuildResult(models.Model):
dest_list = [dest for sublist in [dest_by_builds_ids[rem_id] for rem_id in remaining.ids] for dest in sublist]
_logger.info('(%s) (%s) not deleted because no corresponding build found', label, " ".join(dest_list))
for build in existing:
if fields.Datetime.from_string(build.gc_date) < datetime.datetime.now():
if build.gc_date < fields.datetime.now():
if build.local_state == 'done':
full = build.gc_date + datetime.timedelta(days=(full_gc_days)) < fields.datetime.now()
for db in dest_by_builds_ids[build.id]:
yield db
yield (db, full)
elif build.local_state != 'running':
_logger.warning('db (%s) not deleted because state is not done', " ".join(dest_by_builds_ids[build.id]))
def _local_cleanup(self, force=False):
def _local_cleanup(self, force=False, full=False):
"""
Remove datadir and drop databases of build older than db_gc_days or db_gc_days_child.
If force is set to True, does the same cleaning based on recordset without checking build age.
@ -520,7 +521,7 @@ class BuildResult(models.Model):
for dest in dest_list:
build = self._build_from_dest(dest)
if build and build in self:
yield dest
yield (dest, full)
elif not build:
_logger.info('%s (%s) skipped because not dest format', label, dest)
_filter = filter_ids
@ -529,7 +530,7 @@ class BuildResult(models.Model):
existing_db = list_local_dbs(additionnal_conditions=additionnal_conditions)
for db in _filter(dest_list=existing_db, label='db'):
for db, _ in _filter(dest_list=existing_db, label='db'):
self._logger('Removing database')
self._local_pg_dropdb(db)
@ -537,12 +538,16 @@ class BuildResult(models.Model):
builds_dir = os.path.join(root, 'build')
if force is True:
dests = [build.dest for build in self]
dests = [(build.dest, full) for build in self]
else:
dests = _filter(dest_list=os.listdir(builds_dir), label='workspace')
for dest in dests:
for dest, full in dests:
build_dir = os.path.join(builds_dir, dest)
if full:
_logger.info('Removing build dir "%s"', dest)
shutil.rmtree(build_dir, ignore_errors=True)
continue
for f in os.listdir(build_dir):
path = os.path.join(build_dir, f)
if os.path.isdir(path) and f not in ('logs', 'tests'):
@ -553,7 +558,7 @@ class BuildResult(models.Model):
log_file_path = os.path.join(log_path, f)
if os.path.isdir(log_file_path):
shutil.rmtree(log_file_path)
elif not f.endswith('.txt'):
elif f in ('run.txt', 'wake_up.txt') or not f.endswith('.txt'):
os.unlink(log_file_path)
def _find_port(self):

2
runbot/models/res_config_settings.py
View File

@ -27,6 +27,8 @@ class ResConfigSettings(models.TransientModel):
runbot_db_gc_days = fields.Integer('Days before gc', default=30, config_parameter='runbot.db_gc_days')
runbot_db_gc_days_child = fields.Integer('Days before gc of child', default=15, config_parameter='runbot.db_gc_days_child')
runbot_full_gc_days = fields.Integer('Days before directory removal', default=365, config_parameter='runbot.full_gc_days',
help='Counting from the db removal date')
runbot_pending_warning = fields.Integer('Pending warning limit', default=5, config_parameter='runbot.pending.warning')
runbot_pending_critical = fields.Integer('Pending critical limit', default=5, config_parameter='runbot.pending.critical')

29
runbot/views/res_config_settings_views.xml
View File

@ -12,6 +12,7 @@
<div class="row mt16 o_settings_container">
<div class="col-12 col-lg-6 o_setting_box">
<div class="o_setting_right_pane">
<span class="o_form_label">General Runbot Settings</span>
<div class="content-group">
<label for="runbot_workers" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_workers" style="width: 15%;"/>
@ -24,22 +25,25 @@
<field name="runbot_timeout" style="width: 15%;"/>
<label for="runbot_starting_port" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_starting_port" style="width: 15%;"/>
</div>
</div>
</div>
<div class="col-12 col-lg-6 o_setting_box">
<div class="o_setting_right_pane">
<span class="o_form_label">Git Related Settings</span>
<div class="content-group">
<label for="runbot_do_fetch" class="col-xs-3 o_light_label" style="width: 40%;"/>
<field name="runbot_do_fetch"/>
<label for="runbot_max_age" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_max_age" style="width: 15%;"/>
<label for="runbot_update_frequency" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_update_frequency" style="width: 15%;"/>
<label for="runbot_db_gc_days" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_db_gc_days" style="width: 15%;"/>
<label for="runbot_db_gc_days_child" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_db_gc_days_child" style="width: 15%;"/>
</div>
</div>
</div>
<div class="col-12 col-lg-6 o_setting_box">
<div class="o_setting_right_pane">
<div class="content-group">
<label for="runbot_do_fetch" class="col-xs-3 o_light_label" style="width: 40%;"/>
<field name="runbot_do_fetch"/>
<label for="runbot_do_schedule" class="col-xs-3 o_light_label" style="width: 40%;"/>
<field name="runbot_do_schedule"/>
<label for="runbot_domain" class="col-xs-3 o_light_label" style="width: 40%;"/>
@ -51,6 +55,19 @@
</div>
</div>
</div>
<div class="col-12 col-lg-6 o_setting_box">
<div class="o_setting_right_pane">
<span class="o_form_label">Garbage Collecting Settings</span>
<div class="content-group">
<label for="runbot_db_gc_days" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_db_gc_days" style="width: 15%;"/>
<label for="runbot_db_gc_days_child" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_db_gc_days_child" style="width: 15%;"/>
<label for="runbot_full_gc_days" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_full_gc_days" style="width: 15%;"/>
</div>
</div>
</div>
<label for="runbot_logdb_uri" class="col-xs-3 o_light_label" style="width: 60%;"/>
<field name="runbot_logdb_uri" style="width: 100%;"/>
<label for="runbot_message" class="col-xs-3 o_light_label" style="width: 60%;"/>