diff --git a/runbot_merge/github.py b/runbot_merge/github.py index 3fdec209..1f31e03c 100644 --- a/runbot_merge/github.py +++ b/runbot_merge/github.py @@ -313,92 +313,6 @@ class GH(object): f"Sanity check ref update of {branch}, expected {sha} got {head}" return status - def merge(self, sha, dest, message): - r = self('post', 'merges', json={ - 'base': dest, - 'head': sha, - 'commit_message': message, - }, check={409: MergeError}) - try: - r = r.json() - except Exception: - raise MergeError("got non-JSON reponse from github: %s %s (%s)" % (r.status_code, r.reason, r.text)) - _logger.debug( - "merge(%s, %s (%s), %s) -> %s", - self._repo, dest, r['parents'][0]['sha'], - shorten(message), r['sha'] - ) - return dict(r['commit'], sha=r['sha'], parents=r['parents']) - - def rebase(self, pr, dest, reset=False, commits=None): - """ Rebase pr's commits on top of dest, updates dest unless ``reset`` - is set. - - Returns the hash of the rebased head and a map of all PR commits (to the PR they were rebased to) - """ - logger = _logger.getChild('rebase') - original_head = self.head(dest) - if commits is None: - commits = self.commits(pr) - - logger.debug("rebasing %s, %s on %s (reset=%s, commits=%s)", - self._repo, pr, dest, reset, len(commits)) - - if not commits: - raise MergeError("PR has no commits") - prev = original_head - for original in commits: - if len(original['parents']) != 1: - raise MergeError( - "commits with multiple parents ({sha}) can not be rebased, " - "either fix the branch to remove merges or merge without " - "rebasing".format_map( - original - )) - tmp_msg = 'temp rebasing PR %s (%s)' % (pr, original['sha']) - merged = self.merge(original['sha'], dest, tmp_msg) - - # whichever parent is not original['sha'] should be what dest - # deref'd to, and we want to check that matches the "left parent" we - # expect (either original_head or the previously merged commit) - [base_commit] = (parent['sha'] for parent in merged['parents'] - if parent['sha'] != original['sha']) - if prev != base_commit: - raise MergeError( - f"Inconsistent view of branch {dest} while rebasing " - f"PR {pr} expected commit {prev} but the other parent of " - f"merge commit {merged['sha']} is {base_commit}.\n\n" - f"The branch may be getting concurrently modified." - ) - prev = merged['sha'] - original['new_tree'] = merged['tree']['sha'] - - prev = original_head - mapping = {} - for c in commits: - committer = c['commit']['committer'] - committer.pop('date') - copy = self('post', 'git/commits', json={ - 'message': c['commit']['message'], - 'tree': c['new_tree'], - 'parents': [prev], - 'author': c['commit']['author'], - 'committer': committer, - }, check={409: MergeError}).json() - logger.debug('copied %s to %s (parent: %s)', c['sha'], copy['sha'], prev) - prev = mapping[c['sha']] = copy['sha'] - - if reset: - self.set_ref(dest, original_head) - else: - self.set_ref(dest, prev) - - logger.debug('rebased %s, %s on %s (reset=%s, commits=%s) -> %s', - self._repo, pr, dest, reset, len(commits), - prev) - # prev is updated after each copy so it's the rebased PR head - return prev, mapping - # fetch various bits of issues / prs to load them def pr(self, number): return ( diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py index b2c35d9e..7684249d 100644 --- a/runbot_merge/models/project_freeze/__init__.py +++ b/runbot_merge/models/project_freeze/__init__.py @@ -1,18 +1,19 @@ -import contextlib import enum import itertools import json import logging -import time from collections import Counter +from typing import Dict from markupsafe import Markup from odoo import models, fields, api, Command -from odoo.addons.runbot_merge.exceptions import FastForwardError from odoo.exceptions import UserError from odoo.tools import drop_view_if_exists +from ... import git +from ..pull_requests import Repository + _logger = logging.getLogger(__name__) class FreezeWizard(models.Model): _name = 'runbot_merge.project.freeze' @@ -211,50 +212,50 @@ class FreezeWizard(models.Model): master_name = master.name gh_sessions = {r: r.github() for r in self.project_id.repo_ids} + repos: Dict[Repository, git.Repo] = { + r: git.get_local(r, 'github').check(False) + for r in self.project_id.repo_ids + } + for repo, copy in repos.items(): + copy.fetch(git.source_url(repo, 'github'), '+refs/heads/*:refs/heads/*') # prep new branch (via tmp refs) on every repo - rel_heads = {} + rel_heads: Dict[Repository, str] = {} # store for master heads as odds are high the bump pr(s) will be on the # same repo as one of the release PRs - prevs = {} + prevs: Dict[Repository, str] = {} for rel in self.release_pr_ids: repo_id = rel.repository_id gh = gh_sessions[repo_id] try: prev = prevs[repo_id] = gh.head(master_name) - except Exception: - raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.") + except Exception as e: + raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.") from e - # create the tmp branch to merge the PR into - tmp_branch = f'tmp.{self.branch_name}' try: - gh.set_ref(tmp_branch, prev) - except Exception as err: - raise UserError(f"Unable to create branch {self.branch_name} of repository {repo_id.name}: {err}.") + commits = gh.commits(rel.pr_id.number) + except Exception as e: + raise UserError(f"Unable to fetch commits of release PR {rel.pr_id.display_name}.") from e - rel_heads[repo_id], _ = gh.rebase(rel.pr_id.number, tmp_branch) - time.sleep(1) + rel_heads[repo_id] = repos[repo_id].rebase(prev, commits)[0] # prep bump - bump_heads = {} + bump_heads: Dict[Repository, str] = {} for bump in self.bump_pr_ids: repo_id = bump.repository_id gh = gh_sessions[repo_id] try: prev = prevs[repo_id] = prevs.get(repo_id) or gh.head(master_name) - except Exception: - raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.") + except Exception as e: + raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.") from e - # create the tmp branch to merge the PR into - tmp_branch = f'tmp.{master_name}' try: - gh.set_ref(tmp_branch, prev) - except Exception as err: - raise UserError(f"Unable to create branch {master_name} of repository {repo_id.name}: {err}.") + commits = gh.commits(bump.pr_id.number) + except Exception as e: + raise UserError(f"Unable to fetch commits of bump PR {bump.pr_id.display_name}.") from e - bump_heads[repo_id], _ = gh.rebase(bump.pr_id.number, tmp_branch) - time.sleep(1) + bump_heads[repo_id] = repos[repo_id].rebase(prev, commits)[0] deployed = {} # at this point we've got a bunch of tmp branches with merged release @@ -264,38 +265,39 @@ class FreezeWizard(models.Model): failure = None for rel in self.release_pr_ids: repo_id = rel.repository_id - # helper API currently has no API to ensure we're just creating a - # new branch (as cheaply as possible) so do it by hand - status = None - with contextlib.suppress(Exception): - status = gh_sessions[repo_id].create_ref(self.branch_name, rel_heads[repo_id]) - deployed[rel.pr_id.id] = rel_heads[repo_id] - to_delete.append(repo_id) - if status != 201: + if repos[repo_id].push( + git.source_url(repo_id, 'github'), + f'{rel_heads[repo_id]}:refs/heads/{self.branch_name}', + ).returncode: failure = ('create', repo_id.name, self.branch_name) break + + deployed[rel.pr_id.id] = rel_heads[repo_id] + to_delete.append(repo_id) else: # all release deployments succeeded for bump in self.bump_pr_ids: repo_id = bump.repository_id - try: - gh_sessions[repo_id].fast_forward(master_name, bump_heads[repo_id]) - deployed[bump.pr_id.id] = bump_heads[repo_id] - to_revert.append(repo_id) - except FastForwardError: + if repos[repo_id].push( + git.source_url(repo_id, 'github'), + f'{bump_heads[repo_id]}:refs/heads/{master_name}' + ).returncode: failure = ('fast-forward', repo_id.name, master_name) break + deployed[bump.pr_id.id] = bump_heads[repo_id] + to_revert.append(repo_id) + if failure: addendums = [] # creating the branch failed, try to delete all previous branches failures = [] for prev_id in to_revert: - revert = gh_sessions[prev_id]('PATCH', f'git/refs/heads/{master_name}', json={ - 'sha': prevs[prev_id], - 'force': True - }, check=False) - if not revert.ok: + if repos[prev_id].push( + '-f', + git.source_url(prev_id, 'github'), + f'{prevs[prev_id]}:refs/heads/{master_name}', + ).returncode: failures.append(prev_id.name) if failures: addendums.append( @@ -305,8 +307,10 @@ class FreezeWizard(models.Model): failures.clear() for prev_id in to_delete: - deletion = gh_sessions[prev_id]('DELETE', f'git/refs/heads/{self.branch_name}', check=False) - if not deletion.ok: + if repos[prev_id].push( + git.source_url(prev_id, 'github'), + f':refs/heads/{self.branch_name}' + ).returncode: failures.append(prev_id.name) if failures: addendums.append( @@ -468,7 +472,7 @@ class OpenPRLabels(models.Model): def init(self): super().init() - drop_view_if_exists(self.env.cr, "runbot_merge_freeze_labels"); + drop_view_if_exists(self.env.cr, "runbot_merge_freeze_labels") self.env.cr.execute(""" CREATE VIEW runbot_merge_freeze_labels AS ( SELECT DISTINCT ON (label) diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py index 908b7db4..59b3f932 100644 --- a/runbot_merge/tests/test_multirepo.py +++ b/runbot_merge/tests/test_multirepo.py @@ -5,7 +5,6 @@ source branches). When preparing a staging, we simply want to ensure branch-matched PRs are staged concurrently in all repos """ -import json import time import xmlrpc.client @@ -1256,12 +1255,12 @@ def test_freeze_complete(env, project, repo_a, repo_b, repo_c, users, config): c_b = repo_b.commit('1.1') assert c_b.message.startswith('Release 1.1 (B)') - assert repo_b.read_tree(c_b) == {'f': '1', 'version': ''} + assert repo_b.read_tree(c_b) == {'f': '1', 'version': '1.1'} assert c_b.parents[0] == master_head_b c_c = repo_c.commit('1.1') assert c_c.message.startswith('Release 1.1 (C)') - assert repo_c.read_tree(c_c) == {'f': '2', 'version': ''} + assert repo_c.read_tree(c_c) == {'f': '2', 'version': '1.1'} assert repo_c.commit(c_c.parents[0]).parents[0] == master_head_c @@ -1272,7 +1271,7 @@ def setup_mess(repo_a, repo_b, repo_c): [root, _] = r.make_commits( None, Commit('base', tree={'version': '', 'f': '0'}), - Commit('release 1.0', tree={'version': '1.0'} if r is repo_a else None), + Commit('release 1.0', tree={'version': '1.0'}), ref='heads/1.0' ) master_heads.extend(r.make_commits(root, Commit('other', tree={'f': '1'}), ref='heads/master')) @@ -1294,7 +1293,7 @@ def setup_mess(repo_a, repo_b, repo_c): with repo_b: repo_b.make_commits( master_heads[1], - Commit('Release 1.1 (B)', tree=None), + Commit('Release 1.1 (B)', tree={'version': '1.1'}), ref='heads/release-1.1' ) pr_rel_b = repo_b.make_pr(target='master', head='release-1.1') @@ -1303,7 +1302,7 @@ def setup_mess(repo_a, repo_b, repo_c): pr_other = repo_c.make_pr(target='master', head='whocares') repo_c.make_commits( master_heads[2], - Commit('Release 1.1 (C)', tree=None), + Commit('Release 1.1 (C)', tree={'version': '1.1'}), ref='heads/release-1.1' ) pr_rel_c = repo_c.make_pr(target='master', head='release-1.1') @@ -1431,7 +1430,8 @@ def test_freeze_conflict(env, project, repo_a, repo_b, repo_c, users, config): # create conflicting branch with repo_c: - repo_c.make_ref('heads/1.1', heads[2]) + [c] = repo_c.make_commits(heads[2], Commit("exists", tree={'version': ''})) + repo_c.make_ref('heads/1.1', c) # actually perform the freeze with pytest.raises(xmlrpc.client.Fault) as e: