NextERP/runbot
5
0
Fork 0
mirror of https://github.com/odoo/runbot.git synced 2025-03-15 23:45:44 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

[IMP] runbot: speedup get_refs

Browse Source 
Regarding the number of refs in odoo repo (arround 18 million at this
time), the parsing of the date was significant when filtering old refs.

Using unix time allows a direct comparaison without parsing the date,
and improved performance, going from ~7 seconds to ~1.3 seconds.
This commit is contained in:
Xavier-Do 2022-06-17 15:20:15 +02:00 committed by Christophe Monniez
parent cbfc8401a8
commit cdaae9b3ed
3 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
runbot/models/repo.py
View File

@ -6,7 +6,6 @@ import re
import subprocess
import time
import dateutil
import requests
from pathlib import Path
@ -358,10 +357,11 @@ class Repo(models.Model):
"""
self.ensure_one()
get_ref_time = round(self._get_fetch_head_time(), 4)
commit_limit = time.time() - 60*60*24*max_age
if not self.get_ref_time or get_ref_time > self.get_ref_time:
try:
self.set_ref_time(get_ref_time)
fields = ['refname', 'objectname', 'committerdate:iso8601', 'authorname', 'authoremail', 'subject', 'committername', 'committeremail']
fields = ['refname', 'objectname', 'committerdate:unix', 'authorname', 'authoremail', 'subject', 'committername', 'committeremail']
fmt = "%00".join(["%(" + field + ")" for field in fields])
cmd = ['for-each-ref', '--format', fmt, '--sort=-committerdate', 'refs/*/heads/*']
if any(remote.fetch_pull for remote in self.remote_ids):
@ -371,7 +371,7 @@ class Repo(models.Model):
if not git_refs:
return []
refs = [tuple(field for field in line.split('\x00')) for line in git_refs.split('\n')]
refs = [r for r in refs if dateutil.parser.parse(r[2][:19]) + datetime.timedelta(days=max_age) > datetime.datetime.now() or self.env['runbot.branch'].match_is_base(r[0].split('\n')[-1])]
refs = [r for r in refs if int(r[2]) > commit_limit or self.env['runbot.branch'].match_is_base(r[0].split('\n')[-1])]
if ignore:
refs = [r for r in refs if r[0].split('/')[-1] not in ignore]
return refs
@ -432,7 +432,7 @@ class Repo(models.Model):
'committer': committer,
'committer_email': committer_email,
'subject': subject,
'date': dateutil.parser.parse(date[:19]),
'date': datetime.datetime.fromtimestamp(int(date)),
})
branch.head = commit
if not branch.alive:

4
runbot/tests/common.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
import datetime
import time
from odoo.tests.common import TransactionCase
from unittest.mock import patch, DEFAULT
@ -37,7 +37,7 @@ class RunbotCase(TransactionCase):
self.commit_list[self.repo_server.id] = [(
'refs/%s/heads/%s' % (remote.remote_name, branch_name),
sha or 'd0d0caca',
tstamp or datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
str(tstamp or int(time.time())),
committer,
commiter_email,
subject,

14
runbot/tests/test_repo.py
View File

@ -71,7 +71,7 @@ class TestRepo(RunbotCaseMinimalSetup):
first_commit = [(
'refs/%s/heads/%s' % (self.remote_server_dev.remote_name, branch_name),
'd0d0caca',
datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
str(int(time.time())),
'Marc Bidule',
'<marc.bidule@somewhere.com>',
'Server subject',
@ -99,7 +99,7 @@ class TestRepo(RunbotCaseMinimalSetup):
# create a addons branch in the same bundle
self.commit_list[self.repo_addons.id] = [('refs/%s/heads/%s' % (self.remote_addons_dev.remote_name, branch_name),
'deadbeef',
datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
str(int(time.time())),
'Marc Bidule',
'<marc.bidule@somewhere.com>',
'Addons subject',
@ -127,7 +127,7 @@ class TestRepo(RunbotCaseMinimalSetup):
self.commit_list[self.repo_server.id] += [
('refs/%s/pull/123' % self.remote_server.remote_name,
'd0d0caca',
datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
str(int(time.time())),
'Marc Bidule',
'<marc.bidule@somewhere.com>',
'Another subject',
@ -157,7 +157,7 @@ class TestRepo(RunbotCaseMinimalSetup):
(
'refs/%s/heads/%s' % (self.remote_server_dev.remote_name, branch_name),
'b00b',
datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
str(int(time.time())),
'Marc Bidule',
'<marc.bidule@somewhere.com>',
'A new subject',
@ -167,7 +167,7 @@ class TestRepo(RunbotCaseMinimalSetup):
(
'refs/%s/pull/123' % self.remote_server.remote_name,
'b00b',
datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
str(int(time.time())),
'Marc Bidule',
'<marc.bidule@somewhere.com>',
'A new subject',
@ -207,7 +207,7 @@ class TestRepo(RunbotCaseMinimalSetup):
self.commit_list[self.repo_server.id] = [
('refs/%s/heads/%s' % (self.remote_server_dev.remote_name, branch_name),
'dead1234',
datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
str(int(time.time())),
'Marc Bidule',
'<marc.bidule@somewhere.com>',
'A last subject',
@ -262,7 +262,7 @@ class TestRepo(RunbotCaseMinimalSetup):
for i in range(20005):
self.commit_list[self.repo_server.id].append(['refs/heads/bidon-%05d' % i,
'd0d0caca %s' % i,
datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
str(int(time.time())),
'Marc Bidule',
'<marc.bidule@somewhere.com>',
'A nice subject',