[IMP] runbot: speedup get_refs

Regarding the number of refs in odoo repo (arround 18 million at this
time), the parsing of the date was significant when filtering old refs.

Using unix time allows a direct comparaison without parsing the date,
and improved performance, going from ~7 seconds to ~1.3 seconds.

runbot/models/repo.py

@@ -6,7 +6,6 @@ import re
 import subprocess
 import time
 
-import dateutil
 import requests
 
 from pathlib import Path
@@ -358,10 +357,11 @@ class Repo(models.Model):
         """
         self.ensure_one()
         get_ref_time = round(self._get_fetch_head_time(), 4)
+        commit_limit = time.time() - 60*60*24*max_age
         if not self.get_ref_time or get_ref_time > self.get_ref_time:
             try:
                 self.set_ref_time(get_ref_time)
-                fields = ['refname', 'objectname', 'committerdate:iso8601', 'authorname', 'authoremail', 'subject', 'committername', 'committeremail']
+                fields = ['refname', 'objectname', 'committerdate:unix', 'authorname', 'authoremail', 'subject', 'committername', 'committeremail']
                 fmt = "%00".join(["%(" + field + ")" for field in fields])
                 cmd = ['for-each-ref', '--format', fmt, '--sort=-committerdate', 'refs/*/heads/*']
                 if any(remote.fetch_pull for remote in self.remote_ids):
@@ -371,7 +371,7 @@ class Repo(models.Model):
                 if not git_refs:
                     return []
                 refs = [tuple(field for field in line.split('\x00')) for line in git_refs.split('\n')]
-                refs = [r for r in refs if dateutil.parser.parse(r[2][:19]) + datetime.timedelta(days=max_age) > datetime.datetime.now() or self.env['runbot.branch'].match_is_base(r[0].split('\n')[-1])]
+                refs = [r for r in refs if int(r[2]) > commit_limit or self.env['runbot.branch'].match_is_base(r[0].split('\n')[-1])]
                 if ignore:
                     refs = [r for r in refs if r[0].split('/')[-1] not in ignore]
                 return refs
@@ -432,7 +432,7 @@ class Repo(models.Model):
                         'committer': committer,
                         'committer_email': committer_email,
                         'subject': subject,
-                        'date': dateutil.parser.parse(date[:19]),
+                        'date': datetime.datetime.fromtimestamp(int(date)),
                     })
                 branch.head = commit
                 if not branch.alive:

runbot/tests/common.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import datetime
-
+import time
 from odoo.tests.common import TransactionCase
 from unittest.mock import patch, DEFAULT
 
@@ -37,7 +37,7 @@ class RunbotCase(TransactionCase):
         self.commit_list[self.repo_server.id] = [(
             'refs/%s/heads/%s' % (remote.remote_name, branch_name),
             sha or 'd0d0caca',
-            tstamp or datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
+            str(tstamp or int(time.time())),
             committer,
             commiter_email,
             subject,

runbot/tests/test_repo.py

@@ -71,7 +71,7 @@ class TestRepo(RunbotCaseMinimalSetup):
         first_commit = [(
             'refs/%s/heads/%s' % (self.remote_server_dev.remote_name, branch_name),
             'd0d0caca',
-            datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
+            str(int(time.time())),
             'Marc Bidule',
             '<marc.bidule@somewhere.com>',
             'Server subject',
@@ -99,7 +99,7 @@ class TestRepo(RunbotCaseMinimalSetup):
         # create a addons branch in the same bundle
         self.commit_list[self.repo_addons.id] = [('refs/%s/heads/%s' % (self.remote_addons_dev.remote_name, branch_name),
                                                   'deadbeef',
-                                                  datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
+                                                   str(int(time.time())),
                                                   'Marc Bidule',
                                                   '<marc.bidule@somewhere.com>',
                                                   'Addons subject',
@@ -127,7 +127,7 @@ class TestRepo(RunbotCaseMinimalSetup):
         self.commit_list[self.repo_server.id] += [
             ('refs/%s/pull/123' % self.remote_server.remote_name,
              'd0d0caca',
-             datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
+             str(int(time.time())),
              'Marc Bidule',
              '<marc.bidule@somewhere.com>',
              'Another subject',
@@ -157,7 +157,7 @@ class TestRepo(RunbotCaseMinimalSetup):
             (
                 'refs/%s/heads/%s' % (self.remote_server_dev.remote_name, branch_name),
                 'b00b',
-                datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
+                str(int(time.time())),
                 'Marc Bidule',
                 '<marc.bidule@somewhere.com>',
                 'A new subject',
@@ -167,7 +167,7 @@ class TestRepo(RunbotCaseMinimalSetup):
             (
                 'refs/%s/pull/123' % self.remote_server.remote_name,
                 'b00b',
-                datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
+                str(int(time.time())),
                 'Marc Bidule',
                 '<marc.bidule@somewhere.com>',
                 'A new subject',
@@ -207,7 +207,7 @@ class TestRepo(RunbotCaseMinimalSetup):
         self.commit_list[self.repo_server.id] = [
             ('refs/%s/heads/%s' % (self.remote_server_dev.remote_name, branch_name),
              'dead1234',
-             datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
+             str(int(time.time())),
              'Marc Bidule',
              '<marc.bidule@somewhere.com>',
              'A last subject',
@@ -262,7 +262,7 @@ class TestRepo(RunbotCaseMinimalSetup):
         for i in range(20005):
             self.commit_list[self.repo_server.id].append(['refs/heads/bidon-%05d' % i,
                                                           'd0d0caca %s' % i,
-                                                          datetime.datetime.now().strftime("%Y-%m-%d, %H:%M:%S"),
+                                                          str(int(time.time())),
                                                           'Marc Bidule',
                                                           '<marc.bidule@somewhere.com>',
                                                           'A nice subject',