From cdaae9b3edb404ed68a971a807b01eca8a357a0b Mon Sep 17 00:00:00 2001 From: Xavier-Do Date: Fri, 17 Jun 2022 15:20:15 +0200 Subject: [PATCH] [IMP] runbot: speedup get_refs Regarding the number of refs in odoo repo (arround 18 million at this time), the parsing of the date was significant when filtering old refs. Using unix time allows a direct comparaison without parsing the date, and improved performance, going from ~7 seconds to ~1.3 seconds. --- runbot/models/repo.py | 8 ++++---- runbot/tests/common.py | 4 ++-- runbot/tests/test_repo.py | 14 +++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 65beee3f..f60af754 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -6,7 +6,6 @@ import re import subprocess import time -import dateutil import requests from pathlib import Path @@ -358,10 +357,11 @@ class Repo(models.Model): """ self.ensure_one() get_ref_time = round(self._get_fetch_head_time(), 4) + commit_limit = time.time() - 60*60*24*max_age if not self.get_ref_time or get_ref_time > self.get_ref_time: try: self.set_ref_time(get_ref_time) - fields = ['refname', 'objectname', 'committerdate:iso8601', 'authorname', 'authoremail', 'subject', 'committername', 'committeremail'] + fields = ['refname', 'objectname', 'committerdate:unix', 'authorname', 'authoremail', 'subject', 'committername', 'committeremail'] fmt = "%00".join(["%(" + field + ")" for field in fields]) cmd = ['for-each-ref', '--format', fmt, '--sort=-committerdate', 'refs/*/heads/*'] if any(remote.fetch_pull for remote in self.remote_ids): @@ -371,7 +371,7 @@ class Repo(models.Model): if not git_refs: return [] refs = [tuple(field for field in line.split('\x00')) for line in git_refs.split('\n')] - refs = [r for r in refs if dateutil.parser.parse(r[2][:19]) + datetime.timedelta(days=max_age) > datetime.datetime.now() or self.env['runbot.branch'].match_is_base(r[0].split('\n')[-1])] + refs = [r for r in refs if int(r[2]) > commit_limit or self.env['runbot.branch'].match_is_base(r[0].split('\n')[-1])] if ignore: refs = [r for r in refs if r[0].split('/')[-1] not in ignore] return refs @@ -432,7 +432,7 @@ class Repo(models.Model): 'committer': committer, 'committer_email': committer_email, 'subject': subject, - 'date': dateutil.parser.parse(date[:19]), + 'date': datetime.datetime.fromtimestamp(int(date)), }) branch.head = commit if not branch.alive: diff --git a/runbot/tests/common.py b/runbot/tests/common.py index 9753fc04..6d71d985 100644 --- a/runbot/tests/common.py +++ b/runbot/tests/common.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import datetime - +import time from odoo.tests.common import TransactionCase from unittest.mock import patch, DEFAULT @@ -37,7 +37,7 @@ class RunbotCase(TransactionCase): self.commit_list[self.repo_server.id] = [( 'refs/%s/heads/%s' % (remote.remote_name, branch_name), sha or 'd0d0caca', - tstamp or datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"), + str(tstamp or int(time.time())), committer, commiter_email, subject, diff --git a/runbot/tests/test_repo.py b/runbot/tests/test_repo.py index 6174c309..78e3ca70 100644 --- a/runbot/tests/test_repo.py +++ b/runbot/tests/test_repo.py @@ -71,7 +71,7 @@ class TestRepo(RunbotCaseMinimalSetup): first_commit = [( 'refs/%s/heads/%s' % (self.remote_server_dev.remote_name, branch_name), 'd0d0caca', - datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"), + str(int(time.time())), 'Marc Bidule', '', 'Server subject', @@ -99,7 +99,7 @@ class TestRepo(RunbotCaseMinimalSetup): # create a addons branch in the same bundle self.commit_list[self.repo_addons.id] = [('refs/%s/heads/%s' % (self.remote_addons_dev.remote_name, branch_name), 'deadbeef', - datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"), + str(int(time.time())), 'Marc Bidule', '', 'Addons subject', @@ -127,7 +127,7 @@ class TestRepo(RunbotCaseMinimalSetup): self.commit_list[self.repo_server.id] += [ ('refs/%s/pull/123' % self.remote_server.remote_name, 'd0d0caca', - datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"), + str(int(time.time())), 'Marc Bidule', '', 'Another subject', @@ -157,7 +157,7 @@ class TestRepo(RunbotCaseMinimalSetup): ( 'refs/%s/heads/%s' % (self.remote_server_dev.remote_name, branch_name), 'b00b', - datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"), + str(int(time.time())), 'Marc Bidule', '', 'A new subject', @@ -167,7 +167,7 @@ class TestRepo(RunbotCaseMinimalSetup): ( 'refs/%s/pull/123' % self.remote_server.remote_name, 'b00b', - datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"), + str(int(time.time())), 'Marc Bidule', '', 'A new subject', @@ -207,7 +207,7 @@ class TestRepo(RunbotCaseMinimalSetup): self.commit_list[self.repo_server.id] = [ ('refs/%s/heads/%s' % (self.remote_server_dev.remote_name, branch_name), 'dead1234', - datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"), + str(int(time.time())), 'Marc Bidule', '', 'A last subject', @@ -262,7 +262,7 @@ class TestRepo(RunbotCaseMinimalSetup): for i in range(20005): self.commit_list[self.repo_server.id].append(['refs/heads/bidon-%05d' % i, 'd0d0caca %s' % i, - datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"), + str(int(time.time())), 'Marc Bidule', '', 'A nice subject',