From 3985332a34bf58549238a6d0d4e64f0c64d7967d Mon Sep 17 00:00:00 2001 From: Xavier-Do Date: Tue, 15 Mar 2022 10:35:04 +0100 Subject: [PATCH] [IMP] runbot: commit exports Source cleanup will check in multiple place for potentially used commits. This maked the cleanup logic complex, plus limit the usable commits in python steps. The current use case is to export the mergebase commits. The poposed solution is to save the exported commit and clean this list when the build is done. All commit in this list cannot be cleaned. --- runbot/models/build.py | 5 ++++- runbot/models/commit.py | 14 ++++++++++++-- runbot/models/runbot.py | 9 +++++++-- runbot/security/ir.model.access.csv | 2 ++ 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/runbot/models/build.py b/runbot/models/build.py index da2b8955..2479396f 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -729,6 +729,9 @@ class BuildResult(models.Model): build._github_status() build._run_job() + if build.local_state == 'done': + self.env['runbot.commit.export'].search([('build_id', '=', build.id)]).unlink() + def _run_job(self): # run job for build in self: @@ -794,7 +797,7 @@ class BuildResult(models.Model): if build_export_path in exports: self._log('_checkout', 'Multiple repo have same export path in build, some source may be missing for %s' % build_export_path, level='ERROR') self._kill(result='ko') - exports[build_export_path] = commit.export() + exports[build_export_path] = commit.export(self) checkout_time = time.time() - start if checkout_time > 60: diff --git a/runbot/models/commit.py b/runbot/models/commit.py index 9a9da2bd..cb12b378 100644 --- a/runbot/models/commit.py +++ b/runbot/models/commit.py @@ -52,11 +52,11 @@ class Commit(models.Model): module = os.path.basename(os.path.dirname(manifest_path)) yield (addons_path, module, manifest_file_name) - def export(self): + def export(self, build): """Export a git repo into a sources""" # TODO add automated tests self.ensure_one() - + self.env['runbot.commit.export'].create({'commit_id': self.id, 'build_id': build.id}) export_path = self._source_path() if os.path.isdir(export_path): @@ -228,3 +228,13 @@ class CommitStatus(models.Model): self._cr.after('commit', send_github_status_async) else: send_github_status(self.env) + + +class CommitExport(models.Model): + _name = 'runbot.commit.export' + _description = 'Commit export' + + build_id = fields.Many2one('runbot.build', index=True) + commit_id = fields.Many2one('runbot.commit') + + host = fields.Char(related='build_id.host', store=True) diff --git a/runbot/models/runbot.py b/runbot/models/runbot.py index c99d5fec..c6430fad 100644 --- a/runbot/models/runbot.py +++ b/runbot/models/runbot.py @@ -315,10 +315,15 @@ class Runbot(models.AbstractModel): if self.pool._init: return _logger.info('Source cleaning') - # we can remove a source only if no build are using them as name or rependency_ids aka as commit + + cannot_be_deleted_path = set() + for commit in self.env['runbot.commit.export'].search([('host', '=', fqdn())]).mapped('commit_id'): + cannot_be_deleted_path.add(commit._source_path()) + + + # the following part won't be usefull anymore once runbot.commit.export is populated cannot_be_deleted_builds = self.env['runbot.build'].search([('host', '=', fqdn()), ('local_state', '!=', 'done')]) cannot_be_deleted_builds |= cannot_be_deleted_builds.mapped('params_id.builds_reference_ids') - cannot_be_deleted_path = set() for build in cannot_be_deleted_builds: for build_commit in build.params_id.commit_link_ids: cannot_be_deleted_path.add(build_commit.commit_id._source_path()) diff --git a/runbot/security/ir.model.access.csv b/runbot/security/ir.model.access.csv index 3fec37f8..e906e7b5 100644 --- a/runbot/security/ir.model.access.csv +++ b/runbot/security/ir.model.access.csv @@ -114,3 +114,5 @@ access_runbot_dockerfile_admin,access_runbot_dockerfile_admin,runbot.model_runbo access_runbot_codeowner_admin,runbot_codeowner_admin,runbot.model_runbot_codeowner,runbot.group_runbot_admin,1,1,1,1 access_runbot_codeowner_user,runbot_codeowner_user,runbot.model_runbot_codeowner,group_user,1,0,0,0 + +access_runbot_commit_export_admin,runbot_commit_export_admin,runbot.model_runbot_commit_export,runbot.group_runbot_admin,1,1,1,1