[CHG] runbot_merge: convert freeze wizard to local repo

Probably less necessary than for the regular staging stuff, but might
as well while at it.

Requires updating one of the test to generate a non-ff push, as
O_CREAT doesn't exist at the git level, and the client (and it is
client-side) only protects against force pushes. So there is no way to
trigger an issue with just the creation of the new branch, it needs to
exist *and point to a non-ancestor commit*.

Also remove a sleep in the ref update loop as there are no ref updates
anymore, until the very final sync via git.

NB: maybe it'd be possible to push both bump and release PRs together
for each repo, but getting which update failed in case of failure
seems difficult.

runbot_merge/github.py

@@ -313,92 +313,6 @@ class GH(object):
                     f"Sanity check ref update of {branch}, expected {sha} got {head}"
         return status
 
-    def merge(self, sha, dest, message):
-        r = self('post', 'merges', json={
-            'base': dest,
-            'head': sha,
-            'commit_message': message,
-        }, check={409: MergeError})
-        try:
-            r = r.json()
-        except Exception:
-            raise MergeError("got non-JSON reponse from github: %s %s (%s)" % (r.status_code, r.reason, r.text))
-        _logger.debug(
-            "merge(%s, %s (%s), %s) -> %s",
-            self._repo, dest, r['parents'][0]['sha'],
-            shorten(message), r['sha']
-        )
-        return dict(r['commit'], sha=r['sha'], parents=r['parents'])
-
-    def rebase(self, pr, dest, reset=False, commits=None):
-        """ Rebase pr's commits on top of dest, updates dest unless ``reset``
-        is set.
-
-        Returns the hash of the rebased head and a map of all PR commits (to the PR they were rebased to)
-        """
-        logger = _logger.getChild('rebase')
-        original_head = self.head(dest)
-        if commits is None:
-            commits = self.commits(pr)
-
-        logger.debug("rebasing %s, %s on %s (reset=%s, commits=%s)",
-                     self._repo, pr, dest, reset, len(commits))
-
-        if not commits:
-            raise MergeError("PR has no commits")
-        prev = original_head
-        for original in commits:
-            if len(original['parents']) != 1:
-                raise MergeError(
-                    "commits with multiple parents ({sha}) can not be rebased, "
-                    "either fix the branch to remove merges or merge without "
-                    "rebasing".format_map(
-                    original
-                ))
-            tmp_msg = 'temp rebasing PR %s (%s)' % (pr, original['sha'])
-            merged = self.merge(original['sha'], dest, tmp_msg)
-
-            # whichever parent is not original['sha'] should be what dest
-            # deref'd to, and we want to check that matches the "left parent" we
-            # expect (either original_head or the previously merged commit)
-            [base_commit] = (parent['sha'] for parent in merged['parents']
-                             if parent['sha'] != original['sha'])
-            if prev != base_commit:
-                raise MergeError(
-                    f"Inconsistent view of branch {dest} while rebasing "
-                    f"PR {pr} expected commit {prev} but the other parent of "
-                    f"merge commit {merged['sha']} is {base_commit}.\n\n"
-                    f"The branch may be getting concurrently modified."
-                )
-            prev = merged['sha']
-            original['new_tree'] = merged['tree']['sha']
-
-        prev = original_head
-        mapping = {}
-        for c in commits:
-            committer = c['commit']['committer']
-            committer.pop('date')
-            copy = self('post', 'git/commits', json={
-                'message': c['commit']['message'],
-                'tree': c['new_tree'],
-                'parents': [prev],
-                'author': c['commit']['author'],
-                'committer': committer,
-            }, check={409: MergeError}).json()
-            logger.debug('copied %s to %s (parent: %s)', c['sha'], copy['sha'], prev)
-            prev = mapping[c['sha']] = copy['sha']
-
-        if reset:
-            self.set_ref(dest, original_head)
-        else:
-            self.set_ref(dest, prev)
-
-        logger.debug('rebased %s, %s on %s (reset=%s, commits=%s) -> %s',
-                      self._repo, pr, dest, reset, len(commits),
-                      prev)
-        # prev is updated after each copy so it's the rebased PR head
-        return prev, mapping
-
     # fetch various bits of issues / prs to load them
     def pr(self, number):
         return (

runbot_merge/models/project_freeze/__init__.py

@@ -1,18 +1,19 @@
-import contextlib
 import enum
 import itertools
 import json
 import logging
-import time
 from collections import Counter
+from typing import Dict
 
 from markupsafe import Markup
 
 from odoo import models, fields, api, Command
-from odoo.addons.runbot_merge.exceptions import FastForwardError
 from odoo.exceptions import UserError
 from odoo.tools import drop_view_if_exists
 
+from ... import git
+from ..pull_requests import Repository
+
 _logger = logging.getLogger(__name__)
 class FreezeWizard(models.Model):
     _name = 'runbot_merge.project.freeze'
@@ -211,50 +212,50 @@ class FreezeWizard(models.Model):
         master_name = master.name
 
         gh_sessions = {r: r.github() for r in self.project_id.repo_ids}
+        repos: Dict[Repository, git.Repo] = {
+            r: git.get_local(r, 'github').check(False)
+            for r in self.project_id.repo_ids
+        }
+        for repo, copy in repos.items():
+            copy.fetch(git.source_url(repo, 'github'), '+refs/heads/*:refs/heads/*')
 
         # prep new branch (via tmp refs) on every repo
-        rel_heads = {}
+        rel_heads: Dict[Repository, str] = {}
         # store for master heads as odds are high the bump pr(s) will be on the
         # same repo as one of the release PRs
-        prevs = {}
+        prevs: Dict[Repository, str] = {}
         for rel in self.release_pr_ids:
             repo_id = rel.repository_id
             gh = gh_sessions[repo_id]
             try:
                 prev = prevs[repo_id] = gh.head(master_name)
-            except Exception:
+            except Exception as e:
-                raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.")
+                raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.") from e
 
-            # create the tmp branch to merge the PR into
-            tmp_branch = f'tmp.{self.branch_name}'
             try:
-                gh.set_ref(tmp_branch, prev)
+                commits = gh.commits(rel.pr_id.number)
-            except Exception as err:
+            except Exception as e:
-                raise UserError(f"Unable to create branch {self.branch_name} of repository {repo_id.name}: {err}.")
+                raise UserError(f"Unable to fetch commits of release PR {rel.pr_id.display_name}.") from e
 
-            rel_heads[repo_id], _ = gh.rebase(rel.pr_id.number, tmp_branch)
+            rel_heads[repo_id] = repos[repo_id].rebase(prev, commits)[0]
-            time.sleep(1)
 
         # prep bump
-        bump_heads = {}
+        bump_heads: Dict[Repository, str] = {}
         for bump in self.bump_pr_ids:
             repo_id = bump.repository_id
             gh = gh_sessions[repo_id]
 
             try:
                 prev = prevs[repo_id] = prevs.get(repo_id) or gh.head(master_name)
-            except Exception:
+            except Exception as e:
-                raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.")
+                raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.") from e
 
-            # create the tmp branch to merge the PR into
-            tmp_branch = f'tmp.{master_name}'
             try:
-                gh.set_ref(tmp_branch, prev)
+                commits = gh.commits(bump.pr_id.number)
-            except Exception as err:
+            except Exception as e:
-                raise UserError(f"Unable to create branch {master_name} of repository {repo_id.name}: {err}.")
+                raise UserError(f"Unable to fetch commits of bump PR {bump.pr_id.display_name}.") from e
 
-            bump_heads[repo_id], _ = gh.rebase(bump.pr_id.number, tmp_branch)
+            bump_heads[repo_id] = repos[repo_id].rebase(prev, commits)[0]
-            time.sleep(1)
 
         deployed = {}
         # at this point we've got a bunch of tmp branches with merged release
@@ -264,38 +265,39 @@ class FreezeWizard(models.Model):
         failure = None
         for rel in self.release_pr_ids:
             repo_id = rel.repository_id
-            # helper API currently has no API to ensure we're just creating a
-            # new branch (as cheaply as possible) so do it by hand
-            status = None
-            with contextlib.suppress(Exception):
-                status = gh_sessions[repo_id].create_ref(self.branch_name, rel_heads[repo_id])
-                deployed[rel.pr_id.id] = rel_heads[repo_id]
-                to_delete.append(repo_id)
 
-            if status != 201:
+            if repos[repo_id].push(
+                git.source_url(repo_id, 'github'),
+                f'{rel_heads[repo_id]}:refs/heads/{self.branch_name}',
+            ).returncode:
                 failure = ('create', repo_id.name, self.branch_name)
                 break
+
+            deployed[rel.pr_id.id] = rel_heads[repo_id]
+            to_delete.append(repo_id)
         else: # all release deployments succeeded
             for bump in self.bump_pr_ids:
                 repo_id = bump.repository_id
-                try:
+                if repos[repo_id].push(
-                    gh_sessions[repo_id].fast_forward(master_name, bump_heads[repo_id])
+                    git.source_url(repo_id, 'github'),
-                    deployed[bump.pr_id.id] = bump_heads[repo_id]
+                    f'{bump_heads[repo_id]}:refs/heads/{master_name}'
-                    to_revert.append(repo_id)
+                ).returncode:
-                except FastForwardError:
                     failure = ('fast-forward', repo_id.name, master_name)
                     break
 
+                deployed[bump.pr_id.id] = bump_heads[repo_id]
+                to_revert.append(repo_id)
+
         if failure:
             addendums = []
             # creating the branch failed, try to delete all previous branches
             failures = []
             for prev_id in to_revert:
-                revert = gh_sessions[prev_id]('PATCH', f'git/refs/heads/{master_name}', json={
+                if repos[prev_id].push(
-                    'sha': prevs[prev_id],
+                    '-f',
-                    'force': True
+                    git.source_url(prev_id, 'github'),
-                }, check=False)
+                    f'{prevs[prev_id]}:refs/heads/{master_name}',
-                if not revert.ok:
+                ).returncode:
                     failures.append(prev_id.name)
             if failures:
                 addendums.append(
@@ -305,8 +307,10 @@ class FreezeWizard(models.Model):
                 failures.clear()
 
             for prev_id in to_delete:
-                deletion = gh_sessions[prev_id]('DELETE', f'git/refs/heads/{self.branch_name}', check=False)
+                if repos[prev_id].push(
-                if not deletion.ok:
+                    git.source_url(prev_id, 'github'),
+                    f':refs/heads/{self.branch_name}'
+                ).returncode:
                     failures.append(prev_id.name)
             if failures:
                 addendums.append(
@@ -468,7 +472,7 @@ class OpenPRLabels(models.Model):
 
     def init(self):
         super().init()
-        drop_view_if_exists(self.env.cr, "runbot_merge_freeze_labels");
+        drop_view_if_exists(self.env.cr, "runbot_merge_freeze_labels")
         self.env.cr.execute("""
         CREATE VIEW runbot_merge_freeze_labels AS (
             SELECT DISTINCT ON (label)

runbot_merge/tests/test_multirepo.py

@@ -5,7 +5,6 @@ source branches).
 When preparing a staging, we simply want to ensure branch-matched PRs
 are staged concurrently in all repos
 """
-import json
 import time
 import xmlrpc.client
 
@@ -1256,12 +1255,12 @@ def test_freeze_complete(env, project, repo_a, repo_b, repo_c, users, config):
 
     c_b = repo_b.commit('1.1')
     assert c_b.message.startswith('Release 1.1 (B)')
-    assert repo_b.read_tree(c_b) == {'f': '1', 'version': ''}
+    assert repo_b.read_tree(c_b) == {'f': '1', 'version': '1.1'}
     assert c_b.parents[0] == master_head_b
 
     c_c = repo_c.commit('1.1')
     assert c_c.message.startswith('Release 1.1 (C)')
-    assert repo_c.read_tree(c_c) == {'f': '2', 'version': ''}
+    assert repo_c.read_tree(c_c) == {'f': '2', 'version': '1.1'}
     assert repo_c.commit(c_c.parents[0]).parents[0] == master_head_c
 
 
@@ -1272,7 +1271,7 @@ def setup_mess(repo_a, repo_b, repo_c):
             [root, _] = r.make_commits(
                 None,
                 Commit('base', tree={'version': '', 'f': '0'}),
-                Commit('release 1.0', tree={'version': '1.0'} if r is repo_a else None),
+                Commit('release 1.0', tree={'version': '1.0'}),
                 ref='heads/1.0'
             )
             master_heads.extend(r.make_commits(root, Commit('other', tree={'f': '1'}), ref='heads/master'))
@@ -1294,7 +1293,7 @@ def setup_mess(repo_a, repo_b, repo_c):
     with repo_b:
         repo_b.make_commits(
             master_heads[1],
-            Commit('Release 1.1 (B)', tree=None),
+            Commit('Release 1.1 (B)', tree={'version': '1.1'}),
             ref='heads/release-1.1'
         )
         pr_rel_b = repo_b.make_pr(target='master', head='release-1.1')
@@ -1303,7 +1302,7 @@ def setup_mess(repo_a, repo_b, repo_c):
         pr_other = repo_c.make_pr(target='master', head='whocares')
         repo_c.make_commits(
             master_heads[2],
-            Commit('Release 1.1 (C)', tree=None),
+            Commit('Release 1.1 (C)', tree={'version': '1.1'}),
             ref='heads/release-1.1'
         )
         pr_rel_c = repo_c.make_pr(target='master', head='release-1.1')
@@ -1431,7 +1430,8 @@ def test_freeze_conflict(env, project, repo_a, repo_b, repo_c, users, config):
 
     # create conflicting branch
     with repo_c:
-        repo_c.make_ref('heads/1.1', heads[2])
+        [c] = repo_c.make_commits(heads[2], Commit("exists", tree={'version': ''}))
+        repo_c.make_ref('heads/1.1', c)
 
     # actually perform the freeze
     with pytest.raises(xmlrpc.client.Fault) as e: