NextERP/runbot
5
0
Fork 0
mirror of https://github.com/odoo/runbot.git synced 2025-03-27 13:25:47 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

[CHG] runbot_merge: switch staging from github API to local

Browse Source 
It has been a consideration for a while, but the pain of subtly
interacting with git via the ignominous CLI kept it back. Then ~~the
fire nation attacked~~ github got more and more tight-fisted (and in
some ways less reliable) with their API.

Staging pretty much just interacts with the git database, so it's both
a facultative github operator (it can just interact with git directly)
and a big consumer of API requests (because the git database endpoints
are very low level so it takes quite a bit of work to do anything
especially when high-level operations like rebase have to be
replicated by hand).

Nota: it might be a good idea to use pygit2 instead of the CLI, the
library seems to be typed and might allow doing the processing we need
on a bare repository directly. Which we can do via the plumbing
commands anyway, but still.

Fixes #247
This commit is contained in:
Xavier Morel 2023-08-18 13:51:18 +02:00
parent 5703840b3b
commit 747162d6e1
1 changed files with 25 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
runbot_merge/models/stagings_create.py
View File

@ -349,8 +349,7 @@ def stage_batch(env: api.Environment, prs: PullRequests, staging: StagingState)
new_heads: Dict[PullRequests, str] = {} new_heads: Dict[PullRequests, str] = {}
pr_fields = env['runbot_merge.pull_requests']._fields pr_fields = env['runbot_merge.pull_requests']._fields
for pr in prs: for pr in prs:
gh = staging[pr.repository].gh info = staging[pr.repository]
_logger.info( _logger.info(
"Staging pr %s for target %s; method=%s", "Staging pr %s for target %s; method=%s",
pr.display_name, pr.target.name, pr.display_name, pr.target.name,
@ -358,10 +357,10 @@ def stage_batch(env: api.Environment, prs: PullRequests, staging: StagingState)
) )
target = 'tmp.{}'.format(pr.target.name) target = 'tmp.{}'.format(pr.target.name)
original_head = gh.head(target) original_head = info.gh.head(target)
try: try:
try: try:
method, new_heads[pr] = stage(pr, gh, target, related_prs=(prs - pr)) method, new_heads[pr] = stage(pr, info, target, related_prs=(prs - pr))
_logger.info( _logger.info(
"Staged pr %s to %s by %s: %s -> %s", "Staged pr %s to %s by %s: %s -> %s",
pr.display_name, pr.target.name, method, pr.display_name, pr.target.name, method,
@ -370,7 +369,7 @@ def stage_batch(env: api.Environment, prs: PullRequests, staging: StagingState)
except Exception: except Exception:
# reset the head which failed, as rebase() may have partially # reset the head which failed, as rebase() may have partially
# updated it (despite later steps failing) # updated it (despite later steps failing)
gh.set_ref(target, original_head) info.gh.set_ref(target, original_head)
# then reset every previous update # then reset every previous update
for to_revert in new_heads.keys(): for to_revert in new_heads.keys():
it = staging[to_revert.repository] it = staging[to_revert.repository]
@ -419,9 +418,9 @@ def format_for_difflib(items: Iterator[Tuple[str, object]]) -> Iterator[str]:
Method = Literal['merge', 'rebase-merge', 'rebase-ff', 'squash'] Method = Literal['merge', 'rebase-merge', 'rebase-ff', 'squash']
def stage(pr: PullRequests, gh: github.GH, target: str, related_prs: PullRequests) -> Tuple[Method, str]: def stage(pr: PullRequests, info: StagingSlice, target: str, related_prs: PullRequests) -> Tuple[Method, str]:
# nb: pr_commits is oldest to newest so pr.head is pr_commits[-1] # nb: pr_commits is oldest to newest so pr.head is pr_commits[-1]
_, prdict = gh.pr(pr.number) _, prdict = info.gh.pr(pr.number)
commits = prdict['commits'] commits = prdict['commits']
method: Method = pr.merge_method or ('rebase-ff' if commits == 1 else None) method: Method = pr.merge_method or ('rebase-ff' if commits == 1 else None)
if commits > 50 and method.startswith('rebase'): if commits > 50 and method.startswith('rebase'):
@ -431,7 +430,7 @@ def stage(pr: PullRequests, gh: github.GH, target: str, related_prs: PullRequest
pr, "Merging PRs of 250 or more commits is not supported " pr, "Merging PRs of 250 or more commits is not supported "
"(https://developer.github.com/v3/pulls/#list-commits-on-a-pull-request)" "(https://developer.github.com/v3/pulls/#list-commits-on-a-pull-request)"
) )
pr_commits = gh.commits(pr.number) pr_commits = info.gh.commits(pr.number)
for c in pr_commits: for c in pr_commits:
if not (c['commit']['author']['email'] and c['commit']['committer']['email']): if not (c['commit']['author']['email'] and c['commit']['committer']['email']):
raise exceptions.Unmergeable( raise exceptions.Unmergeable(
@ -475,7 +474,7 @@ def stage(pr: PullRequests, gh: github.GH, target: str, related_prs: PullRequest
if pr.reviewed_by and pr.reviewed_by.name == pr.reviewed_by.github_login: if pr.reviewed_by and pr.reviewed_by.name == pr.reviewed_by.github_login:
# XXX: find other trigger(s) to sync github name? # XXX: find other trigger(s) to sync github name?
gh_name = gh.user(pr.reviewed_by.github_login)['name'] gh_name = info.gh.user(pr.reviewed_by.github_login)['name']
if gh_name: if gh_name:
pr.reviewed_by.name = gh_name pr.reviewed_by.name = gh_name
@ -488,9 +487,9 @@ def stage(pr: PullRequests, gh: github.GH, target: str, related_prs: PullRequest
fn = stage_rebase_ff fn = stage_rebase_ff
case 'squash': case 'squash':
fn = stage_squash fn = stage_squash
return method, fn(pr, gh, target, pr_commits, related_prs=related_prs) return method, fn(pr, info, target, pr_commits, related_prs=related_prs)
def stage_squash(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str: def stage_squash(pr: PullRequests, info: StagingSlice, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
msg = pr._build_merge_message(pr, related_prs=related_prs) msg = pr._build_merge_message(pr, related_prs=related_prs)
authorship = {} authorship = {}
@ -516,15 +515,15 @@ def stage_squash(pr: PullRequests, gh: github.GH, target: str, commits: List[git
authorship['committer'] = {'name': name, 'email': email} authorship['committer'] = {'name': name, 'email': email}
# should committers also be added to co-authors? # should committers also be added to co-authors?
original_head = gh.head(target) original_head = info.gh.head(target)
merge_tree = gh.merge(pr.head, target, 'temp merge')['tree']['sha'] merge_tree = info.gh.merge(pr.head, target, 'temp merge')['tree']['sha']
head = gh('post', 'git/commits', json={ head = info.gh('post', 'git/commits', json={
**authorship, **authorship,
'message': str(msg), 'message': str(msg),
'tree': merge_tree, 'tree': merge_tree,
'parents': [original_head], 'parents': [original_head],
}).json()['sha'] }).json()['sha']
gh.set_ref(target, head) info.gh.set_ref(target, head)
commits_map = {c['sha']: head for c in commits} commits_map = {c['sha']: head for c in commits}
commits_map[''] = head commits_map[''] = head
@ -532,25 +531,25 @@ def stage_squash(pr: PullRequests, gh: github.GH, target: str, commits: List[git
return head return head
def stage_rebase_ff(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str: def stage_rebase_ff(pr: PullRequests, info: StagingSlice, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
# updates head commit with PR number (if necessary) then rebases # updates head commit with PR number (if necessary) then rebases
# on top of target # on top of target
msg = pr._build_merge_message(commits[-1]['commit']['message'], related_prs=related_prs) msg = pr._build_merge_message(commits[-1]['commit']['message'], related_prs=related_prs)
commits[-1]['commit']['message'] = str(msg) commits[-1]['commit']['message'] = str(msg)
add_self_references(pr, commits[:-1]) add_self_references(pr, commits[:-1])
head, mapping = gh.rebase(pr.number, target, commits=commits) head, mapping = info.gh.rebase(pr.number, target, commits=commits)
pr.commits_map = json.dumps({**mapping, '': head}) pr.commits_map = json.dumps({**mapping, '': head})
return head return head
def stage_rebase_merge(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str : def stage_rebase_merge(pr: PullRequests, info: StagingSlice, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str :
add_self_references(pr, commits) add_self_references(pr, commits)
h, mapping = gh.rebase(pr.number, target, reset=True, commits=commits) h, mapping = info.gh.rebase(pr.number, target, reset=True, commits=commits)
msg = pr._build_merge_message(pr, related_prs=related_prs) msg = pr._build_merge_message(pr, related_prs=related_prs)
merge_head = gh.merge(h, target, str(msg))['sha'] merge_head = info.gh.merge(h, target, str(msg))['sha']
pr.commits_map = json.dumps({**mapping, '': merge_head}) pr.commits_map = json.dumps({**mapping, '': merge_head})
return merge_head return merge_head
def stage_merge(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str: def stage_merge(pr: PullRequests, info: StagingSlice, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
pr_head = commits[-1] # oldest to newest pr_head = commits[-1] # oldest to newest
base_commit = None base_commit = None
head_parents = {p['sha'] for p in pr_head['parents']} head_parents = {p['sha'] for p in pr_head['parents']}
@ -573,18 +572,18 @@ def stage_merge(pr: PullRequests, gh: github.GH, target: str, commits: List[gith
if base_commit: if base_commit:
# replicate pr_head with base_commit replaced by # replicate pr_head with base_commit replaced by
# the current head # the current head
original_head = gh.head(target) original_head = info.gh.head(target)
merge_tree = gh.merge(pr_head['sha'], target, 'temp merge')['tree']['sha'] merge_tree = info.gh.merge(pr_head['sha'], target, 'temp merge')['tree']['sha']
new_parents = [original_head] + list(head_parents - {base_commit}) new_parents = [original_head] + list(head_parents - {base_commit})
msg = pr._build_merge_message(pr_head['commit']['message'], related_prs=related_prs) msg = pr._build_merge_message(pr_head['commit']['message'], related_prs=related_prs)
copy = gh('post', 'git/commits', json={ copy = info.gh('post', 'git/commits', json={
'message': str(msg), 'message': str(msg),
'tree': merge_tree, 'tree': merge_tree,
'author': pr_head['commit']['author'], 'author': pr_head['commit']['author'],
'committer': pr_head['commit']['committer'], 'committer': pr_head['commit']['committer'],
'parents': new_parents, 'parents': new_parents,
}).json() }).json()
gh.set_ref(target, copy['sha']) info.gh.set_ref(target, copy['sha'])
# merge commit *and old PR head* map to the pr head replica # merge commit *and old PR head* map to the pr head replica
commits_map[''] = commits_map[pr_head['sha']] = copy['sha'] commits_map[''] = commits_map[pr_head['sha']] = copy['sha']
pr.commits_map = json.dumps(commits_map) pr.commits_map = json.dumps(commits_map)
@ -592,7 +591,7 @@ def stage_merge(pr: PullRequests, gh: github.GH, target: str, commits: List[gith
else: else:
# otherwise do a regular merge # otherwise do a regular merge
msg = pr._build_merge_message(pr) msg = pr._build_merge_message(pr)
merge_head = gh.merge(pr.head, target, str(msg))['sha'] merge_head = info.gh.merge(pr.head, target, str(msg))['sha']
# and the merge commit is the normal merge head # and the merge commit is the normal merge head
commits_map[''] = merge_head commits_map[''] = merge_head
pr.commits_map = json.dumps(commits_map) pr.commits_map = json.dumps(commits_map)