[IMP] runbot: store times in other tables.

Sometimes, sheduler may have a hard time to create build.
The transaction can be verry long when there are many repo and
a lot of new commits. Writing get_ref_time on repo will fail
due to concurrent update rollbacking the whole transaction.

This is supposedly because of hook occuring during the transaction.
With this new model, hook will only perform an insert, and shouldn't
interfer with ref_times.

runbot/models/repo.py

@@ -41,8 +41,8 @@ class runbot_repo(models.Model):
                              ('hook', 'Hook')],
                             default='poll',
                             string="Mode", required=True, help="hook: Wait for webhook on /runbot/hook/<id> i.e. github push event")
-    hook_time = fields.Float('Last hook time')
-    get_ref_time = fields.Float('Last refs db update')
+    hook_time = fields.Float('Last hook time', compute='_compute_hook_time')
+    get_ref_time = fields.Float('Last refs db update', compute='_compute_get_ref_time')
     duplicate_id = fields.Many2one('runbot.repo', 'Duplicate repo', help='Repository for finding duplicate builds')
     modules = fields.Char("Modules to install", help="Comma-separated list of modules to install and test.")
     modules_auto = fields.Selection([('none', 'None (only explicit modules list)'),
@@ -77,6 +77,58 @@ class runbot_repo(models.Model):
         for repo in self:
             repo.repo_config_id = repo.config_id
 
+    def _compute_get_ref_time(self):
+        self.env.cr.execute("""
+            SELECT repo_id, time FROM runbot_repo_reftime
+            WHERE id IN (
+                SELECT max(id) FROM runbot_repo_reftime 
+                WHERE repo_id = any(%s) GROUP BY repo_id
+            )
+        """, [self.ids])
+        times = dict(self.env.cr.fetchall())
+        for repo in self:
+            repo.get_ref_time = times.get(repo.id, 0)
+
+    def _compute_hook_time(self):
+        self.env.cr.execute("""
+            SELECT repo_id, time FROM runbot_repo_hooktime
+            WHERE id IN (
+                SELECT max(id) FROM runbot_repo_hooktime 
+                WHERE repo_id = any(%s) GROUP BY repo_id
+            )
+        """, [self.ids])
+        times = dict(self.env.cr.fetchall())
+
+        for repo in self:
+            repo.hook_time = times.get(repo.id, 0)
+
+    def write(self, values):
+        # hooktime and reftime table are here to avoid sql update on repo.
+        # using inverse will still trigger write_date and write_uid update.
+        # this hack allows to avoid that
+
+        hook_time = values.pop('hook_time', None)
+        get_ref_time = values.pop('get_ref_time', None)
+        for repo in self:
+            if hook_time:
+                self.env['runbot.repo.hooktime'].create({'time': hook_time, 'repo_id': repo.id})
+            if get_ref_time:
+                self.env['runbot.repo.reftime'].create({'time': get_ref_time, 'repo_id': repo.id})
+        if values:
+            super().write(values)
+
+    def _gc_times(self):
+        self.env.cr.execute("""
+            DELETE from runbot_repo_reftime WHERE id NOT IN (
+                SELECT max(id) FROM runbot_repo_reftime GROUP BY repo_id
+            )
+        """)
+        self.env.cr.execute("""
+            DELETE from runbot_repo_hooktime WHERE id NOT IN (
+                SELECT max(id) FROM runbot_repo_hooktime GROUP BY repo_id
+            )
+        """)
+
     def _root(self):
         """Return root directory of repository"""
         default = os.path.join(os.path.dirname(__file__), '../static')
@@ -665,3 +717,19 @@ class runbot_repo(models.Model):
         except:
             _logger.error('An exception occured while cleaning sources')
             pass
+
+
+    class RefTime(models.Model):
+        _name = "runbot.repo.reftime"
+        _log_access = False
+
+        time = fields.Float('Time', index=True, required=True)
+        repo_id = fields.Many2one('runbot.repo', 'Repository', required=True, ondelete='cascade')
+
+
+    class HookTime(models.Model):
+        _name = "runbot.repo.hooktime"
+        _log_access = False
+
+        time = fields.Float('Time')
+        repo_id = fields.Many2one('runbot.repo', 'Repository', required=True, ondelete='cascade')

runbot/security/ir.model.access.csv

@@ -30,4 +30,7 @@ access_runbot_host_user,runbot_host_user,runbot.model_runbot_host,group_user,1,0
 access_runbot_host_manager,runbot_host_manager,runbot.model_runbot_host,runbot.group_runbot_admin,1,1,1,1
 
 access_runbot_error_log_user,runbot_error_log_user,runbot.model_runbot_error_log,group_user,1,0,0,0
-access_runbot_error_log_manager,runbot_error_log_manager,runbot.model_runbot_error_log,runbot.group_runbot_admin,1,1,1,1
+access_runbot_error_log_manager,runbot_error_log_manager,runbot.model_runbot_error_log,runbot.group_runbot_admin,1,1,1,1
+
+access_runbot_repo_hooktime,runbot_repo_hooktime,runbot.model_runbot_repo_hooktime,group_user,1,0,0,0
+access_runbot_repo_reftime,runbot_repo_reftime,runbot.model_runbot_repo_reftime,group_user,1,0,0,0

runbot/tests/test_repo.py

@@ -176,6 +176,39 @@ class Test_Repo(RunbotCase):
 
         _logger.info('Create pending builds took: %ssec', (time.time() - inserted_time))
 
+    def test_times(self):
+        def _test_times(model, field_name):
+            repo1 = self.Repo.create({'name': 'bla@example.com:foo/bar'})
+            repo2 = self.Repo.create({'name': 'bla@example.com:foo2/bar2'})
+            count = self.cr.sql_log_count
+            repo1[field_name] = 1.1
+            self.assertEqual(self.cr.sql_log_count - count, 1, "Only one insert should have been triggered")
+            repo2[field_name] = 1.2
+            self.assertEqual(len(self.env[model].search([])), 2)
+            self.assertEqual(repo1[field_name], 1.1)
+            self.assertEqual(repo2[field_name], 1.2)
+
+            repo1[field_name] = 1.3
+            repo2[field_name] = 1.4
+
+            self.assertEqual(len(self.env[model].search([])), 4)
+            self.assertEqual(repo1[field_name], 1.3)
+            self.assertEqual(repo2[field_name], 1.4)
+
+            self.Repo.invalidate_cache()
+            self.assertEqual(repo1[field_name], 1.3)
+            self.assertEqual(repo2[field_name], 1.4)
+
+            self.Repo._gc_times()
+
+            self.assertEqual(len(self.env[model].search([])), 2)
+            self.assertEqual(repo1[field_name], 1.3)
+            self.assertEqual(repo2[field_name], 1.4)
+
+        _test_times('runbot.repo.hooktime', 'hook_time')
+        _test_times('runbot.repo.reftime', 'get_ref_time')
+
+
 
 class Test_Github(TransactionCase):
     def test_github(self):