NextERP/runbot
5
0
Fork 0
mirror of https://github.com/odoo/runbot.git synced 2025-03-15 23:45:44 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

[CHG] runbot_merge: convert freeze wizard to local repo

Browse Source 
Probably less necessary than for the regular staging stuff, but might
as well while at it.

Requires updating one of the test to generate a non-ff push, as
O_CREAT doesn't exist at the git level, and the client (and it is
client-side) only protects against force pushes. So there is no way to
trigger an issue with just the creation of the new branch, it needs to
exist *and point to a non-ancestor commit*.

Also remove a sleep in the ref update loop as there are no ref updates
anymore, until the very final sync via git.

NB: maybe it'd be possible to push both bump and release PRs together
for each repo, but getting which update failed in case of failure
seems difficult.
This commit is contained in:
Xavier Morel 2023-08-23 14:09:32 +02:00
parent 85a7890023
commit 4d2c0f86e1
3 changed files with 56 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
runbot_merge/github.py
View File

@ -313,92 +313,6 @@ class GH(object):
f"Sanity check ref update of {branch}, expected {sha} got {head}"
return status
def merge(self, sha, dest, message):
r = self('post', 'merges', json={
'base': dest,
'head': sha,
'commit_message': message,
}, check={409: MergeError})
try:
r = r.json()
except Exception:
raise MergeError("got non-JSON reponse from github: %s %s (%s)" % (r.status_code, r.reason, r.text))
_logger.debug(
"merge(%s, %s (%s), %s) -> %s",
self._repo, dest, r['parents'][0]['sha'],
shorten(message), r['sha']
)
return dict(r['commit'], sha=r['sha'], parents=r['parents'])
def rebase(self, pr, dest, reset=False, commits=None):
""" Rebase pr's commits on top of dest, updates dest unless ``reset``
is set.
Returns the hash of the rebased head and a map of all PR commits (to the PR they were rebased to)
"""
logger = _logger.getChild('rebase')
original_head = self.head(dest)
if commits is None:
commits = self.commits(pr)
logger.debug("rebasing %s, %s on %s (reset=%s, commits=%s)",
self._repo, pr, dest, reset, len(commits))
if not commits:
raise MergeError("PR has no commits")
prev = original_head
for original in commits:
if len(original['parents']) != 1:
raise MergeError(
"commits with multiple parents ({sha}) can not be rebased, "
"either fix the branch to remove merges or merge without "
"rebasing".format_map(
original
))
tmp_msg = 'temp rebasing PR %s (%s)' % (pr, original['sha'])
merged = self.merge(original['sha'], dest, tmp_msg)
# whichever parent is not original['sha'] should be what dest
# deref'd to, and we want to check that matches the "left parent" we
# expect (either original_head or the previously merged commit)
[base_commit] = (parent['sha'] for parent in merged['parents']
if parent['sha'] != original['sha'])
if prev != base_commit:
raise MergeError(
f"Inconsistent view of branch {dest} while rebasing "
f"PR {pr} expected commit {prev} but the other parent of "
f"merge commit {merged['sha']} is {base_commit}.\n\n"
f"The branch may be getting concurrently modified."
)
prev = merged['sha']
original['new_tree'] = merged['tree']['sha']
prev = original_head
mapping = {}
for c in commits:
committer = c['commit']['committer']
committer.pop('date')
copy = self('post', 'git/commits', json={
'message': c['commit']['message'],
'tree': c['new_tree'],
'parents': [prev],
'author': c['commit']['author'],
'committer': committer,
}, check={409: MergeError}).json()
logger.debug('copied %s to %s (parent: %s)', c['sha'], copy['sha'], prev)
prev = mapping[c['sha']] = copy['sha']
if reset:
self.set_ref(dest, original_head)
else:
self.set_ref(dest, prev)
logger.debug('rebased %s, %s on %s (reset=%s, commits=%s) -> %s',
self._repo, pr, dest, reset, len(commits),
prev)
# prev is updated after each copy so it's the rebased PR head
return prev, mapping
# fetch various bits of issues / prs to load them
def pr(self, number):
return (

94
runbot_merge/models/project_freeze/__init__.py
View File

@ -1,18 +1,19 @@
import contextlib
import enum
import itertools
import json
import logging
import time
from collections import Counter
from typing import Dict
from markupsafe import Markup
from odoo import models, fields, api, Command
from odoo.addons.runbot_merge.exceptions import FastForwardError
from odoo.exceptions import UserError
from odoo.tools import drop_view_if_exists
from ... import git
from ..pull_requests import Repository
_logger = logging.getLogger(__name__)
class FreezeWizard(models.Model):
_name = 'runbot_merge.project.freeze'
@ -211,50 +212,50 @@ class FreezeWizard(models.Model):
master_name = master.name
gh_sessions = {r: r.github() for r in self.project_id.repo_ids}
repos: Dict[Repository, git.Repo] = {
r: git.get_local(r, 'github').check(False)
for r in self.project_id.repo_ids
}
for repo, copy in repos.items():
copy.fetch(git.source_url(repo, 'github'), '+refs/heads/*:refs/heads/*')
# prep new branch (via tmp refs) on every repo
rel_heads = {}
rel_heads: Dict[Repository, str] = {}
# store for master heads as odds are high the bump pr(s) will be on the
# same repo as one of the release PRs
prevs = {}
prevs: Dict[Repository, str] = {}
for rel in self.release_pr_ids:
repo_id = rel.repository_id
gh = gh_sessions[repo_id]
try:
prev = prevs[repo_id] = gh.head(master_name)
except Exception:
raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.")
except Exception as e:
raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.") from e
# create the tmp branch to merge the PR into
tmp_branch = f'tmp.{self.branch_name}'
try:
gh.set_ref(tmp_branch, prev)
except Exception as err:
raise UserError(f"Unable to create branch {self.branch_name} of repository {repo_id.name}: {err}.")
commits = gh.commits(rel.pr_id.number)
except Exception as e:
raise UserError(f"Unable to fetch commits of release PR {rel.pr_id.display_name}.") from e
rel_heads[repo_id], _ = gh.rebase(rel.pr_id.number, tmp_branch)
time.sleep(1)
rel_heads[repo_id] = repos[repo_id].rebase(prev, commits)[0]
# prep bump
bump_heads = {}
bump_heads: Dict[Repository, str] = {}
for bump in self.bump_pr_ids:
repo_id = bump.repository_id
gh = gh_sessions[repo_id]
try:
prev = prevs[repo_id] = prevs.get(repo_id) or gh.head(master_name)
except Exception:
raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.")
except Exception as e:
raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.") from e
# create the tmp branch to merge the PR into
tmp_branch = f'tmp.{master_name}'
try:
gh.set_ref(tmp_branch, prev)
except Exception as err:
raise UserError(f"Unable to create branch {master_name} of repository {repo_id.name}: {err}.")
commits = gh.commits(bump.pr_id.number)
except Exception as e:
raise UserError(f"Unable to fetch commits of bump PR {bump.pr_id.display_name}.") from e
bump_heads[repo_id], _ = gh.rebase(bump.pr_id.number, tmp_branch)
time.sleep(1)
bump_heads[repo_id] = repos[repo_id].rebase(prev, commits)[0]
deployed = {}
# at this point we've got a bunch of tmp branches with merged release
@ -264,38 +265,39 @@ class FreezeWizard(models.Model):
failure = None
for rel in self.release_pr_ids:
repo_id = rel.repository_id
# helper API currently has no API to ensure we're just creating a
# new branch (as cheaply as possible) so do it by hand
status = None
with contextlib.suppress(Exception):
status = gh_sessions[repo_id].create_ref(self.branch_name, rel_heads[repo_id])
deployed[rel.pr_id.id] = rel_heads[repo_id]
to_delete.append(repo_id)
if status != 201:
if repos[repo_id].push(
git.source_url(repo_id, 'github'),
f'{rel_heads[repo_id]}:refs/heads/{self.branch_name}',
).returncode:
failure = ('create', repo_id.name, self.branch_name)
break
deployed[rel.pr_id.id] = rel_heads[repo_id]
to_delete.append(repo_id)
else: # all release deployments succeeded
for bump in self.bump_pr_ids:
repo_id = bump.repository_id
try:
gh_sessions[repo_id].fast_forward(master_name, bump_heads[repo_id])
deployed[bump.pr_id.id] = bump_heads[repo_id]
to_revert.append(repo_id)
except FastForwardError:
if repos[repo_id].push(
git.source_url(repo_id, 'github'),
f'{bump_heads[repo_id]}:refs/heads/{master_name}'
).returncode:
failure = ('fast-forward', repo_id.name, master_name)
break
deployed[bump.pr_id.id] = bump_heads[repo_id]
to_revert.append(repo_id)
if failure:
addendums = []
# creating the branch failed, try to delete all previous branches
failures = []
for prev_id in to_revert:
revert = gh_sessions[prev_id]('PATCH', f'git/refs/heads/{master_name}', json={
'sha': prevs[prev_id],
'force': True
}, check=False)
if not revert.ok:
if repos[prev_id].push(
'-f',
git.source_url(prev_id, 'github'),
f'{prevs[prev_id]}:refs/heads/{master_name}',
).returncode:
failures.append(prev_id.name)
if failures:
addendums.append(
@ -305,8 +307,10 @@ class FreezeWizard(models.Model):
failures.clear()
for prev_id in to_delete:
deletion = gh_sessions[prev_id]('DELETE', f'git/refs/heads/{self.branch_name}', check=False)
if not deletion.ok:
if repos[prev_id].push(
git.source_url(prev_id, 'github'),
f':refs/heads/{self.branch_name}'
).returncode:
failures.append(prev_id.name)
if failures:
addendums.append(
@ -468,7 +472,7 @@ class OpenPRLabels(models.Model):
def init(self):
super().init()
drop_view_if_exists(self.env.cr, "runbot_merge_freeze_labels");
drop_view_if_exists(self.env.cr, "runbot_merge_freeze_labels")
self.env.cr.execute("""
CREATE VIEW runbot_merge_freeze_labels AS (
SELECT DISTINCT ON (label)

14
runbot_merge/tests/test_multirepo.py
View File

@ -5,7 +5,6 @@ source branches).
When preparing a staging, we simply want to ensure branch-matched PRs
are staged concurrently in all repos
"""
import json
import time
import xmlrpc.client
@ -1256,12 +1255,12 @@ def test_freeze_complete(env, project, repo_a, repo_b, repo_c, users, config):
c_b = repo_b.commit('1.1')
assert c_b.message.startswith('Release 1.1 (B)')
assert repo_b.read_tree(c_b) == {'f': '1', 'version': ''}
assert repo_b.read_tree(c_b) == {'f': '1', 'version': '1.1'}
assert c_b.parents[0] == master_head_b
c_c = repo_c.commit('1.1')
assert c_c.message.startswith('Release 1.1 (C)')
assert repo_c.read_tree(c_c) == {'f': '2', 'version': ''}
assert repo_c.read_tree(c_c) == {'f': '2', 'version': '1.1'}
assert repo_c.commit(c_c.parents[0]).parents[0] == master_head_c
@ -1272,7 +1271,7 @@ def setup_mess(repo_a, repo_b, repo_c):
[root, _] = r.make_commits(
None,
Commit('base', tree={'version': '', 'f': '0'}),
Commit('release 1.0', tree={'version': '1.0'} if r is repo_a else None),
Commit('release 1.0', tree={'version': '1.0'}),
ref='heads/1.0'
)
master_heads.extend(r.make_commits(root, Commit('other', tree={'f': '1'}), ref='heads/master'))
@ -1294,7 +1293,7 @@ def setup_mess(repo_a, repo_b, repo_c):
with repo_b:
repo_b.make_commits(
master_heads[1],
Commit('Release 1.1 (B)', tree=None),
Commit('Release 1.1 (B)', tree={'version': '1.1'}),
ref='heads/release-1.1'
)
pr_rel_b = repo_b.make_pr(target='master', head='release-1.1')
@ -1303,7 +1302,7 @@ def setup_mess(repo_a, repo_b, repo_c):
pr_other = repo_c.make_pr(target='master', head='whocares')
repo_c.make_commits(
master_heads[2],
Commit('Release 1.1 (C)', tree=None),
Commit('Release 1.1 (C)', tree={'version': '1.1'}),
ref='heads/release-1.1'
)
pr_rel_c = repo_c.make_pr(target='master', head='release-1.1')
@ -1431,7 +1430,8 @@ def test_freeze_conflict(env, project, repo_a, repo_b, repo_c, users, config):
# create conflicting branch
with repo_c:
repo_c.make_ref('heads/1.1', heads[2])
[c] = repo_c.make_commits(heads[2], Commit("exists", tree={'version': ''}))
repo_c.make_ref('heads/1.1', c)
# actually perform the freeze
with pytest.raises(xmlrpc.client.Fault) as e: