[IMP] runbot: commit exports

Source cleanup will check in multiple place for potentially used
commits. This maked the cleanup logic complex, plus limit the usable
commits in python steps. The current use case is to export the mergebase
commits.

The poposed solution is to save the exported commit and clean this list
when the build is done. All commit in this list cannot be cleaned.

runbot/models/build.py

@@ -729,6 +729,9 @@ class BuildResult(models.Model):
                 build._github_status()
             build._run_job()
 
+            if build.local_state == 'done':
+                self.env['runbot.commit.export'].search([('build_id', '=', build.id)]).unlink()
+
     def _run_job(self):
         # run job
         for build in self:
@@ -794,7 +797,7 @@ class BuildResult(models.Model):
             if build_export_path in exports:
                 self._log('_checkout', 'Multiple repo have same export path in build, some source may be missing for %s' % build_export_path, level='ERROR')
                 self._kill(result='ko')
-            exports[build_export_path] = commit.export()
+            exports[build_export_path] = commit.export(self)
 
         checkout_time = time.time() - start
         if checkout_time > 60:

runbot/models/commit.py

@@ -52,11 +52,11 @@ class Commit(models.Model):
                     module = os.path.basename(os.path.dirname(manifest_path))
                     yield (addons_path, module, manifest_file_name)
 
-    def export(self):
+    def export(self, build):
         """Export a git repo into a sources"""
         #  TODO add automated tests
         self.ensure_one()
-
+        self.env['runbot.commit.export'].create({'commit_id': self.id, 'build_id': build.id})
         export_path = self._source_path()
 
         if os.path.isdir(export_path):
@@ -228,3 +228,13 @@ class CommitStatus(models.Model):
                 self._cr.after('commit', send_github_status_async)
             else:
                 send_github_status(self.env)
+
+
+class CommitExport(models.Model):
+    _name = 'runbot.commit.export'
+    _description = 'Commit export'
+
+    build_id = fields.Many2one('runbot.build', index=True)
+    commit_id = fields.Many2one('runbot.commit')
+
+    host = fields.Char(related='build_id.host', store=True)

runbot/models/runbot.py

@@ -315,10 +315,15 @@ class Runbot(models.AbstractModel):
             if self.pool._init:
                 return
             _logger.info('Source cleaning')
-            # we can remove a source only if no build are using them as name or rependency_ids aka as commit
+
+            cannot_be_deleted_path = set()
+            for commit in self.env['runbot.commit.export'].search([('host', '=', fqdn())]).mapped('commit_id'):
+                cannot_be_deleted_path.add(commit._source_path())
+
+
+            # the following part won't be usefull anymore once runbot.commit.export is populated
             cannot_be_deleted_builds = self.env['runbot.build'].search([('host', '=', fqdn()), ('local_state', '!=', 'done')])
             cannot_be_deleted_builds |= cannot_be_deleted_builds.mapped('params_id.builds_reference_ids')
-            cannot_be_deleted_path = set()
             for build in cannot_be_deleted_builds:
                 for build_commit in build.params_id.commit_link_ids:
                     cannot_be_deleted_path.add(build_commit.commit_id._source_path())

runbot/security/ir.model.access.csv

@@ -114,3 +114,5 @@ access_runbot_dockerfile_admin,access_runbot_dockerfile_admin,runbot.model_runbo
 
 access_runbot_codeowner_admin,runbot_codeowner_admin,runbot.model_runbot_codeowner,runbot.group_runbot_admin,1,1,1,1
 access_runbot_codeowner_user,runbot_codeowner_user,runbot.model_runbot_codeowner,group_user,1,0,0,0
+
+access_runbot_commit_export_admin,runbot_commit_export_admin,runbot.model_runbot_commit_export,runbot.group_runbot_admin,1,1,1,1