[IMP] runbot: decouple github hooks processing

Decouples the hook processing from the controller, this ensures that
latency for the controller stays low while allowing more complex logic
during hook handling.

runbot/controllers/hook.py

@@ -68,17 +68,10 @@ class Hook(http.Controller):
         # force update of dependencies too in case a hook is lost
         if not payload or event == 'push':
             remote.repo_id._set_hook_time(time.time())
-        elif event == 'pull_request':
-            pr_number = payload.get('pull_request', {}).get('number', '')
-            branch = request.env['runbot.branch'].sudo().search([('remote_id', '=', remote.id), ('name', '=', pr_number)])
-            branch._recompute_infos(payload.get('pull_request', {}))
-            if payload.get('action') in ('synchronize', 'opened', 'reopened'):
-                remote.repo_id._set_hook_time(time.time())
-            # remaining recurrent actions: labeled, review_requested, review_request_removed
-        elif event == 'delete':
-            if payload.get('ref_type') == 'branch':
-                branch_ref = payload.get('ref')
-                _logger.info('Branch %s in repo %s was deleted', branch_ref, remote.repo_id.name)
-                branch = request.env['runbot.branch'].sudo().search([('remote_id', '=', remote.id), ('name', '=', branch_ref)])
-                branch.alive = False
+        else:
+            request.env['runbot.repo.hook.payload'].sudo().create({
+                'remote_id': remote.id,
+                'payload': payload,
+                'event': event,
+            })
         return ""

runbot/models/repo.py

@@ -762,3 +762,41 @@ class HookTime(models.Model):
 
     time = fields.Float('Time')
     repo_id = fields.Many2one('runbot.repo', 'Repository', required=True, ondelete='cascade')
+
+
+class HookPayload(models.Model):
+    _name = 'runbot.repo.hook.payload'
+    _description = "Repo hook payloads"
+    _log_access = False
+
+    create_date = fields.Datetime(default=fields.Datetime.now, required=True)
+    remote_id = fields.Many2one('runbot.remote', required=True, ondelete='cascade')
+    event = fields.Char(required=True)
+    payload = JsonDictField(required=True)
+
+    @api.model
+    def _process_all(self):
+        create_date_limit = self.env.cr.now()
+        while (records := self.search([('create_date', '<=', create_date_limit)], limit=1000)):
+            for record in records:
+                record._process_one()
+            records.unlink()
+            self.env.cr.commit()
+
+    def _process_one(self):
+        self.ensure_one()
+        payload = self.payload.dict
+        remote = self.remote_id
+        if self.event == 'pull_request':
+            pr_number = payload.get('pull_request', {}).get('number', '')
+            branch = self.env['runbot.branch'].sudo().search([('remote_id', '=', remote.id), ('name', '=', pr_number)])
+            branch._recompute_infos(payload.get('pull_request', {}))
+            if payload.get('action') in ('synchronize', 'opened', 'reopened'):
+                remote.repo_id._set_hook_time(time.time())
+            # remaining recurrent actions: labeled, review_requested, review_request_removed
+        elif self.event == 'delete':
+            if payload.get('ref_type') == 'branch':
+                branch_ref = payload.get('ref')
+                _logger.info('Branch %s in repo %s was deleted', branch_ref, remote.repo_id.name)
+                branch = self.env['runbot.branch'].sudo().search([('remote_id', '=', remote.id), ('name', '=', branch_ref)])
+                branch.alive = False

runbot/models/runbot.py

@@ -245,6 +245,8 @@ class Runbot(models.AbstractModel):
 
     def _fetch_loop_turn(self, host, pull_info_failures, default_sleep=1):
         with self._manage_host_exception(host) as manager:
+            self.env['runbot.repo.hook.payload']._process_all()
+
             repos = self.env['runbot.repo'].search([('mode', '!=', 'disabled')])
             processing_batch = self.env['runbot.batch'].search([('state', 'in', ('preparing', 'ready'))], order='id asc')
             preparing_batch = processing_batch.filtered(lambda b: b.state == 'preparing')

runbot/security/ir.model.access.csv

@@ -64,6 +64,9 @@ access_runbot_error_log_manager,runbot_error_log_manager,runbot.model_runbot_err
 access_runbot_repo_hooktime,runbot_repo_hooktime,runbot.model_runbot_repo_hooktime,group_user,1,0,0,0
 access_runbot_repo_referencetime,runbot_repo_referencetime,runbot.model_runbot_repo_reftime,group_user,1,0,0,0
 
+access_runbot_repo_payload_user,runbot_repo_payload_user,runbot.model_runbot_repo_hook_payload,group_user,0,0,0,0
+access_runbot_repo_payload_admin,runbot_repo_payload_admin,runbot.model_runbot_repo_hook_payload,runbot.group_runbot_admin,1,1,1,1
+
 access_runbot_build_stat_user,runbot_build_stat_user,runbot.model_runbot_build_stat,group_user,1,0,0,0
 access_runbot_build_stat_admin,runbot_build_stat_admin,runbot.model_runbot_build_stat,runbot.group_runbot_admin,1,1,1,1