[IMP] runbot: store times in other tables.

Sometimes, sheduler may have a hard time to create build.
The transaction can be verry long when there are many repo and
a lot of new commits. Writing get_ref_time on repo will fail
due to concurrent update rollbacking the whole transaction.

This is supposedly because of hook occuring during the transaction.
With this new model, hook will only perform an insert, and shouldn't
interfer with ref_times.
This commit is contained in:
Xavier-Do 2019-12-06 10:17:40 +01:00 committed by XavierDo
parent 3f806416d6
commit e5a2f98c59
3 changed files with 107 additions and 3 deletions

View File

@ -41,8 +41,8 @@ class runbot_repo(models.Model):
('hook', 'Hook')],
default='poll',
string="Mode", required=True, help="hook: Wait for webhook on /runbot/hook/<id> i.e. github push event")
hook_time = fields.Float('Last hook time')
get_ref_time = fields.Float('Last refs db update')
hook_time = fields.Float('Last hook time', compute='_compute_hook_time')
get_ref_time = fields.Float('Last refs db update', compute='_compute_get_ref_time')
duplicate_id = fields.Many2one('runbot.repo', 'Duplicate repo', help='Repository for finding duplicate builds')
modules = fields.Char("Modules to install", help="Comma-separated list of modules to install and test.")
modules_auto = fields.Selection([('none', 'None (only explicit modules list)'),
@ -77,6 +77,58 @@ class runbot_repo(models.Model):
for repo in self:
repo.repo_config_id = repo.config_id
def _compute_get_ref_time(self):
self.env.cr.execute("""
SELECT repo_id, time FROM runbot_repo_reftime
WHERE id IN (
SELECT max(id) FROM runbot_repo_reftime
WHERE repo_id = any(%s) GROUP BY repo_id
)
""", [self.ids])
times = dict(self.env.cr.fetchall())
for repo in self:
repo.get_ref_time = times.get(repo.id, 0)
def _compute_hook_time(self):
self.env.cr.execute("""
SELECT repo_id, time FROM runbot_repo_hooktime
WHERE id IN (
SELECT max(id) FROM runbot_repo_hooktime
WHERE repo_id = any(%s) GROUP BY repo_id
)
""", [self.ids])
times = dict(self.env.cr.fetchall())
for repo in self:
repo.hook_time = times.get(repo.id, 0)
def write(self, values):
# hooktime and reftime table are here to avoid sql update on repo.
# using inverse will still trigger write_date and write_uid update.
# this hack allows to avoid that
hook_time = values.pop('hook_time', None)
get_ref_time = values.pop('get_ref_time', None)
for repo in self:
if hook_time:
self.env['runbot.repo.hooktime'].create({'time': hook_time, 'repo_id': repo.id})
if get_ref_time:
self.env['runbot.repo.reftime'].create({'time': get_ref_time, 'repo_id': repo.id})
if values:
super().write(values)
def _gc_times(self):
self.env.cr.execute("""
DELETE from runbot_repo_reftime WHERE id NOT IN (
SELECT max(id) FROM runbot_repo_reftime GROUP BY repo_id
)
""")
self.env.cr.execute("""
DELETE from runbot_repo_hooktime WHERE id NOT IN (
SELECT max(id) FROM runbot_repo_hooktime GROUP BY repo_id
)
""")
def _root(self):
"""Return root directory of repository"""
default = os.path.join(os.path.dirname(__file__), '../static')
@ -665,3 +717,19 @@ class runbot_repo(models.Model):
except:
_logger.error('An exception occured while cleaning sources')
pass
class RefTime(models.Model):
_name = "runbot.repo.reftime"
_log_access = False
time = fields.Float('Time', index=True, required=True)
repo_id = fields.Many2one('runbot.repo', 'Repository', required=True, ondelete='cascade')
class HookTime(models.Model):
_name = "runbot.repo.hooktime"
_log_access = False
time = fields.Float('Time')
repo_id = fields.Many2one('runbot.repo', 'Repository', required=True, ondelete='cascade')

View File

@ -30,4 +30,7 @@ access_runbot_host_user,runbot_host_user,runbot.model_runbot_host,group_user,1,0
access_runbot_host_manager,runbot_host_manager,runbot.model_runbot_host,runbot.group_runbot_admin,1,1,1,1
access_runbot_error_log_user,runbot_error_log_user,runbot.model_runbot_error_log,group_user,1,0,0,0
access_runbot_error_log_manager,runbot_error_log_manager,runbot.model_runbot_error_log,runbot.group_runbot_admin,1,1,1,1
access_runbot_error_log_manager,runbot_error_log_manager,runbot.model_runbot_error_log,runbot.group_runbot_admin,1,1,1,1
access_runbot_repo_hooktime,runbot_repo_hooktime,runbot.model_runbot_repo_hooktime,group_user,1,0,0,0
access_runbot_repo_reftime,runbot_repo_reftime,runbot.model_runbot_repo_reftime,group_user,1,0,0,0
1 id name model_id:id group_id:id perm_read perm_write perm_create perm_unlink
30
31
32
33
34
35
36

View File

@ -176,6 +176,39 @@ class Test_Repo(RunbotCase):
_logger.info('Create pending builds took: %ssec', (time.time() - inserted_time))
def test_times(self):
def _test_times(model, field_name):
repo1 = self.Repo.create({'name': 'bla@example.com:foo/bar'})
repo2 = self.Repo.create({'name': 'bla@example.com:foo2/bar2'})
count = self.cr.sql_log_count
repo1[field_name] = 1.1
self.assertEqual(self.cr.sql_log_count - count, 1, "Only one insert should have been triggered")
repo2[field_name] = 1.2
self.assertEqual(len(self.env[model].search([])), 2)
self.assertEqual(repo1[field_name], 1.1)
self.assertEqual(repo2[field_name], 1.2)
repo1[field_name] = 1.3
repo2[field_name] = 1.4
self.assertEqual(len(self.env[model].search([])), 4)
self.assertEqual(repo1[field_name], 1.3)
self.assertEqual(repo2[field_name], 1.4)
self.Repo.invalidate_cache()
self.assertEqual(repo1[field_name], 1.3)
self.assertEqual(repo2[field_name], 1.4)
self.Repo._gc_times()
self.assertEqual(len(self.env[model].search([])), 2)
self.assertEqual(repo1[field_name], 1.3)
self.assertEqual(repo2[field_name], 1.4)
_test_times('runbot.repo.hooktime', 'hook_time')
_test_times('runbot.repo.reftime', 'get_ref_time')
class Test_Github(TransactionCase):
def test_github(self):