mirror of
https://github.com/odoo/runbot.git
synced 2025-03-15 23:45:44 +07:00
[ADD] runbot_merge: staging reifier
Add experimental support for creating submodule-based commits for stagings (and batches), and pushing those in ancillary repositories. Fixes #768
This commit is contained in:
Xavier Morel
parent
0fd254b5fe
commit
2ace06957a
@ -10,6 +10,7 @@
|
||||
'models/crons/git_maintenance.xml',
|
||||
'models/crons/cleanup_scratch_branches.xml',
|
||||
'models/crons/issues_closer.xml',
|
||||
'models/crons/staging_reifier.xml',
|
||||
'data/runbot_merge.pull_requests.feedback.template.csv',
|
||||
'views/res_partner.xml',
|
||||
'views/runbot_merge_project.xml',
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
from . import git_maintenance
|
||||
from . import cleanup_scratch_branches
|
||||
from . import issues_closer
|
||||
from . import staging_reifier
|
||||
|
||||
350
runbot_merge/models/crons/staging_reifier.py
Normal file
350
runbot_merge/models/crons/staging_reifier.py
Normal file
@ -0,0 +1,350 @@
|
||||
""" Implements the reification of stagings into cross-repo git commits
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import datetime
|
||||
import itertools
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from subprocess import PIPE
|
||||
from types import SimpleNamespace
|
||||
from typing import TypedDict
|
||||
|
||||
from odoo import api, fields, models
|
||||
from ....runbot_merge import git
|
||||
from ..batch import Batch
|
||||
from ..pull_requests import Branch, StagingCommits
|
||||
|
||||
|
||||
class Project(models.Model):
|
||||
_inherit = 'runbot_merge.project'
|
||||
|
||||
repo_flat_layout = fields.Char()
|
||||
repo_pythonpath_layout = fields.Char()
|
||||
|
||||
|
||||
class Stagings(models.Model):
|
||||
_inherit = 'runbot_merge.stagings'
|
||||
|
||||
# can not make these required because we need to have created the staging
|
||||
# in order to know what its id is (also makes handling of batches easier)
|
||||
# and NOT NULL can't be deferred
|
||||
hash_flat_layout = fields.Char()
|
||||
hash_pythonpath_layout = fields.Char()
|
||||
|
||||
id: int
|
||||
target: Branch
|
||||
batch_ids: Batch
|
||||
commits: StagingCommits
|
||||
staged_at: datetime.datetime
|
||||
|
||||
|
||||
class Repository(models.Model):
|
||||
_inherit = 'runbot_merge.repository'
|
||||
|
||||
name: str
|
||||
pythonpath_location = fields.Char(
|
||||
help="Where the submodule should be located in the pythonpath layout, "
|
||||
"if empty defaults to the root, unless `pythonpath_link_path` is "
|
||||
"set, then defaults to `.repos`."
|
||||
""
|
||||
"%(fullname)s, %(owner)s, and %(name)s are available as"
|
||||
"placeholders in the path",
|
||||
)
|
||||
pythonpath_link_path = fields.Char(
|
||||
help="Where the repository should be symlinked in the pythonpath layout,"
|
||||
" leave empty to not symlink."
|
||||
""
|
||||
"%(fullname)s, %(owner)s, and %(name)s are available as"
|
||||
"placeholders in the path",
|
||||
)
|
||||
pythonpath_link_target = fields.Char(
|
||||
help="Where the symlink should point to inside the repository, "
|
||||
"leave empty for the root",
|
||||
)
|
||||
|
||||
class Reifier(models.Model):
|
||||
_name = 'runbot_merge.staging.reifier'
|
||||
_description = "transcriptor of staging into cross-repo git commits"
|
||||
_order = 'create_date asc, id asc'
|
||||
|
||||
staging_id: Stagings = fields.Many2one('runbot_merge.stagings', required=True)
|
||||
previous_staging_id: Stagings = fields.Many2one('runbot_merge.stagings')
|
||||
|
||||
@api.model_create_multi
|
||||
def create(self, vals_list):
|
||||
self.env.ref('runbot_merge.staging_reifier')._trigger()
|
||||
return super().create(vals_list)
|
||||
|
||||
def _run(self):
|
||||
projects = self.env['runbot_merge.project']
|
||||
branches = set()
|
||||
while r := self.search([], limit=1):
|
||||
reify(r.staging_id, r.previous_staging_id)
|
||||
projects |= r.staging_id.target.project_id
|
||||
branches.add(r.staging_id.target)
|
||||
r.unlink()
|
||||
self.env.cr.commit()
|
||||
|
||||
for project in projects:
|
||||
if name_flat := project.repo_flat_layout:
|
||||
git.get_local(SimpleNamespace(
|
||||
name=name_flat,
|
||||
project_id=project,
|
||||
)).push('origin', 'refs/heads/*:refs/heads/*')
|
||||
if name_pythonpath := project.repo_pythonpath_layout:
|
||||
git.get_local(SimpleNamespace(
|
||||
name=name_pythonpath,
|
||||
project_id=project,
|
||||
)).push('origin', 'refs/heads/*:refs/heads/*')
|
||||
def reify(staging: Stagings, prev: Stagings) -> None:
|
||||
project = staging.target.project_id
|
||||
repos = project.repo_ids.having_branch(staging.target)
|
||||
|
||||
repo_flat = None
|
||||
commit_flat = prev.hash_flat_layout
|
||||
if name_flat := project.repo_flat_layout:
|
||||
repo_flat = git.get_local(SimpleNamespace(
|
||||
name=name_flat,
|
||||
project_id=project,
|
||||
)).with_config(check=True, encoding="utf-8", stdout=PIPE)
|
||||
|
||||
repo_pythonpath = None
|
||||
commit_pythonpath = prev.hash_pythonpath_layout
|
||||
if name_pythonpath := project.repo_pythonpath_layout:
|
||||
repo_pythonpath = git.get_local(SimpleNamespace(
|
||||
name=name_pythonpath,
|
||||
project_id=project,
|
||||
)).with_config(check=True, encoding="utf-8", stdout=PIPE)
|
||||
|
||||
if prev:
|
||||
commits = {c.repository_id.name: c.commit_id.sha for c in prev.commits}
|
||||
else:
|
||||
# if there is no `prev` then we need to run the staging in reverse in
|
||||
# order to get all the "root" commits for the staging, then we need to
|
||||
# get their parents in order to know what state the repos were in
|
||||
# before the staging, and *that* lets us figure out the per-batch state
|
||||
commits = {c.repository_id.name: c.commit_id.sha for c in staging.commits}
|
||||
|
||||
# need to work off of the PR commits, because if no PR touches a
|
||||
# repository then `commits` could hold *the first commit in the
|
||||
# repository*, in which case we don't want to remove it
|
||||
pr_commits = {}
|
||||
for batch in reversed(staging.batch_ids):
|
||||
# FIXME: broken if the PR has more than one commit and was rebased...
|
||||
pr_commits.update(
|
||||
(pr.repository.name, json.loads(pr.commits_map)[''])
|
||||
for pr in batch.prs
|
||||
)
|
||||
|
||||
for r, c in pr_commits.items():
|
||||
repo = git.get_local(SimpleNamespace(
|
||||
name=r,
|
||||
project_id=project,
|
||||
)).with_config(encoding="utf-8", stdout=PIPE)
|
||||
if parent := repo.rev_parse('--revs-only', c + '~').stdout.strip():
|
||||
commits[r] = parent
|
||||
else:
|
||||
# if a PR's commit has no parent then the PR itself introduced
|
||||
# the repo (what?) in which case we don't have the repo at the
|
||||
# staging and should ignore it
|
||||
del commits[r]
|
||||
|
||||
for batch in staging.batch_ids:
|
||||
commits.update(
|
||||
(pr.repository.name, json.loads(pr.commits_map)[''])
|
||||
for pr in batch.prs
|
||||
)
|
||||
prs = batch.prs.sorted('repository').mapped('display_name')
|
||||
meta = json.dumps({
|
||||
'staging': staging.id,
|
||||
'batch': batch.id,
|
||||
'label': batch.name,
|
||||
'pull_requests': prs,
|
||||
'commits': commits,
|
||||
})
|
||||
|
||||
message = f"{staging.id}.{batch.id}: {batch.name}\n\n- " + "\n- ".join(prs)
|
||||
if repo_flat:
|
||||
tree_flat = layout_flat(
|
||||
repo_flat, {'meta.json': GitBlob(meta)}, repos, commits)
|
||||
commit_flat = repo_flat.commit_tree(
|
||||
tree=tree_flat,
|
||||
message=message,
|
||||
parents=[commit_flat] if commit_flat else [],
|
||||
author=(project.github_name, project.github_email, staging.staged_at.isoformat(timespec='seconds')),
|
||||
committer=(project.github_name, project.github_email, staging.staged_at.isoformat(timespec='seconds')),
|
||||
).stdout.strip()
|
||||
if repo_pythonpath:
|
||||
tree_pythonpath = layout_pythonpath(
|
||||
repo_pythonpath,
|
||||
{'meta.json': GitBlob(meta)},
|
||||
repos,
|
||||
commits,
|
||||
)
|
||||
commit_pythonpath = repo_pythonpath.commit_tree(
|
||||
tree=tree_pythonpath,
|
||||
message=message,
|
||||
parents=[commit_pythonpath] if commit_pythonpath else [],
|
||||
author=('robodoo', 'robodoo@odoo.com', staging.staged_at.isoformat(timespec='seconds')),
|
||||
committer=('robodoo', 'robodoo@odoo.com', staging.staged_at.isoformat(timespec='seconds')),
|
||||
).stdout.strip()
|
||||
|
||||
if repo_flat:
|
||||
repo_flat.update_ref(
|
||||
f'refs/heads/{staging.target.name}',
|
||||
commit_flat,
|
||||
prev.hash_flat_layout or '',
|
||||
)
|
||||
if repo_pythonpath:
|
||||
repo_pythonpath.update_ref(
|
||||
f'refs/heads/{staging.target.name}',
|
||||
commit_pythonpath,
|
||||
prev.hash_pythonpath_layout or '',
|
||||
)
|
||||
staging.write({
|
||||
'hash_flat_layout': commit_flat,
|
||||
'hash_pythonpath_layout': commit_pythonpath,
|
||||
})
|
||||
|
||||
|
||||
def make_submodule(repo: str, path: str) -> str:
|
||||
_, n = repo.split('/')
|
||||
return f"""[submodule.{n}]
|
||||
path = {path}
|
||||
url = https://github.com/{repo}
|
||||
"""
|
||||
|
||||
def layout_flat(
|
||||
repo: git.Repo,
|
||||
tree_init: dict[str, GitObject],
|
||||
repos: Repository,
|
||||
commits: dict[str, str],
|
||||
) -> str:
|
||||
gitmodules = "\n".join(
|
||||
make_submodule(repo.name, repo.name.split('/')[-1])
|
||||
for repo in repos
|
||||
if repo.name in commits
|
||||
)
|
||||
|
||||
return GitTree({
|
||||
**tree_init,
|
||||
".gitmodules": GitBlob(gitmodules),
|
||||
**{
|
||||
repo.name.split('/')[-1]: GitCommit(cc)
|
||||
for repo in repos
|
||||
if (cc := commits.get(repo.name))
|
||||
}
|
||||
}).write(repo, 0)
|
||||
|
||||
def layout_pythonpath(
|
||||
repo: git.Repo,
|
||||
tree_init: dict[str, GitObject],
|
||||
repos: Repository,
|
||||
commits: dict[str, str],
|
||||
) -> str:
|
||||
gitmodules = []
|
||||
tree = GitTree(tree_init)
|
||||
for repo_id in repos:
|
||||
commit = commits.get(repo_id.name)
|
||||
if commit is None:
|
||||
continue
|
||||
|
||||
owner, name = repo_id.name.split('/')
|
||||
tmpl = {'fullname': repo_id.name, 'owner': owner, 'name': name}
|
||||
if p := repo_id.pythonpath_location:
|
||||
path = p % tmpl
|
||||
elif repo_id.pythonpath_link_path:
|
||||
path = ".repos/%(name)s" % tmpl
|
||||
else:
|
||||
path = '%(name)s' % tmpl
|
||||
link = (repo_id.pythonpath_link_path or '') % tmpl
|
||||
target = repo_id.pythonpath_link_target or ''
|
||||
|
||||
tree.set(path, GitCommit(commit))
|
||||
gitmodules.append(make_submodule(repo_id.name, path))
|
||||
|
||||
if link:
|
||||
if target:
|
||||
target = f"{path}/{target}"
|
||||
else:
|
||||
target = path
|
||||
tree.set(link, GitLink(target))
|
||||
|
||||
tree.set(".gitmodules", GitBlob("\n".join(gitmodules)))
|
||||
return tree.write(repo, 0)
|
||||
|
||||
class Commit(TypedDict):
|
||||
staging_id: int
|
||||
commit_id: int
|
||||
repository_id: int
|
||||
|
||||
class GitObject(metaclass=abc.ABCMeta):
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def mode(self) -> str:
|
||||
...
|
||||
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def type(self) -> str:
|
||||
...
|
||||
|
||||
@abc.abstractmethod
|
||||
def write(self, repo: git.Repo, depth: int) -> str:
|
||||
...
|
||||
|
||||
def assert_tree(self) -> GitTree:
|
||||
assert isinstance(self, GitTree)
|
||||
return self
|
||||
|
||||
@dataclass
|
||||
class GitCommit(GitObject):
|
||||
sha: str
|
||||
mode = "160000"
|
||||
type = "commit"
|
||||
|
||||
def write(self, repo: git.Repo, _: int) -> str:
|
||||
return self.sha
|
||||
|
||||
@dataclass
|
||||
class GitTree(GitObject):
|
||||
members: dict[str, GitObject]
|
||||
mode = "40000"
|
||||
type = "tree"
|
||||
|
||||
def set(self, path: str, obj: GitObject) -> None:
|
||||
p, _, target = path.rpartition('/')
|
||||
assert target, f"{path!r} can't be empty"
|
||||
if p:
|
||||
for part in p.split('/'):
|
||||
self = self.members.setdefault(part, GitTree({})).assert_tree()
|
||||
self.members[target] = obj
|
||||
|
||||
def write(self, repo: git.Repo, depth: int) -> str:
|
||||
assert all(self.members), \
|
||||
f"One of the tree entries has an empty filename {self.members}"
|
||||
return repo.with_config(input="".join(
|
||||
f"{obj.mode} {obj.type} {obj.write(repo, depth+1)}\t{name}\n"
|
||||
for name, obj in self.members.items()
|
||||
)).mktree().stdout.strip()
|
||||
|
||||
@dataclass
|
||||
class GitLink(GitObject):
|
||||
reference: str
|
||||
mode = "120000"
|
||||
type = "blob"
|
||||
|
||||
def write(self, repo: git.Repo, depth: int) -> str:
|
||||
target = "/".join(itertools.repeat("..", depth)) + "/" + self.reference
|
||||
return repo.with_config(input=target).hash_object("--stdin", "-w").stdout.strip()
|
||||
|
||||
@dataclass
|
||||
class GitBlob(GitObject):
|
||||
content: str
|
||||
mode = "100644"
|
||||
type = "blob"
|
||||
|
||||
def write(self, repo: git.Repo, _: int) -> str:
|
||||
return repo.with_config(input=self.content).hash_object('--stdin', '-w').stdout.strip()
|
||||
23
runbot_merge/models/crons/staging_reifier.xml
Normal file
23
runbot_merge/models/crons/staging_reifier.xml
Normal file
@ -0,0 +1,23 @@
|
||||
<odoo>
|
||||
<record id="access_staging_reifier" model="ir.model.access">
|
||||
<field name="name">Access to staging reifier is useless</field>
|
||||
<field name="model_id" ref="model_runbot_merge_staging_reifier"/>
|
||||
<field name="perm_read">0</field>
|
||||
<field name="perm_create">0</field>
|
||||
<field name="perm_write">0</field>
|
||||
<field name="perm_unlink">0</field>
|
||||
</record>
|
||||
|
||||
<record model="ir.cron" id="staging_reifier">
|
||||
<field name="name">Reifiy stagings to cross-repo commits</field>
|
||||
<field name="model_id" ref="model_runbot_merge_staging_reifier"/>
|
||||
<field name="state">code</field>
|
||||
<field name="code">model._run()</field>
|
||||
<!--
|
||||
nota: even though this is only triggered, numbercall has to be
|
||||
non-zero because the counter is taken in account by cron triggers
|
||||
-->
|
||||
<field name="numbercall">-1</field>
|
||||
<field name="doall" eval="False"/>
|
||||
</record>
|
||||
</odoo>
|
||||
@ -2435,6 +2435,37 @@ class Stagings(models.Model):
|
||||
})
|
||||
if self.issues_to_close:
|
||||
self.env['runbot_merge.issues_closer'].create(self.issues_to_close)
|
||||
|
||||
# FIXME: error prone, should probably store the previous
|
||||
# staging on creation instead?
|
||||
last2 = self.with_context(active_search=False).search([
|
||||
('state', '=', 'success'),
|
||||
('target', '=', self.target.id),
|
||||
], order='id desc', limit=2)
|
||||
if len(last2) == 2:
|
||||
_self, previous = last2
|
||||
elif self.target != project.branch_ids[:1]:
|
||||
_self = last2
|
||||
previous = self.search([
|
||||
('state', '=', 'success'),
|
||||
('target', '=', project.branch_ids[:1].id),
|
||||
('active', '=', False),
|
||||
], order='id desc', limit=1)
|
||||
else:
|
||||
# no previous branch
|
||||
_self = last2
|
||||
previous = self.browse()
|
||||
if self == _self:
|
||||
self.env['runbot_merge.staging.reifier'].create({
|
||||
'previous_staging_id': previous.id,
|
||||
'staging_id': self.id,
|
||||
})
|
||||
else:
|
||||
_logger.warning(
|
||||
"Got different self (%s) and last success (%s)",
|
||||
self,
|
||||
_self,
|
||||
)
|
||||
finally:
|
||||
self.write({'active': False})
|
||||
elif self.state == 'failure' or self.is_timed_out():
|
||||
|
||||
95
runbot_merge/tests/test_staging_reifier.py
Normal file
95
runbot_merge/tests/test_staging_reifier.py
Normal file
@ -0,0 +1,95 @@
|
||||
import copy
|
||||
import pprint
|
||||
|
||||
import pytest
|
||||
|
||||
from utils import Commit, to_pr
|
||||
|
||||
|
||||
def test_basic(make_repo, project, env, setreviewers, config, users, partners, pytestconfig):
|
||||
repos = {}
|
||||
#region project setup
|
||||
project.repo_pythonpath_layout = make_repo('pythonpath', hooks=False).name
|
||||
for name, conf in zip('abcd', [
|
||||
# flat-style repo (odoo/documentation)
|
||||
{},
|
||||
# flat repo with symlink into (odoo/odoo)
|
||||
{
|
||||
'pythonpath_location': '%(name)s',
|
||||
'pythonpath_link_path': 'community/odoo/addons',
|
||||
'pythonpath_link_target': 'b',
|
||||
},
|
||||
# modules directory style
|
||||
{'pythonpath_location': '%(name)s/odoo/addons'},
|
||||
# upgrade style
|
||||
{
|
||||
'pythonpath_link_path': '%(name)s/odoo/upgrade',
|
||||
'pythonpath_link_target': 'd',
|
||||
},
|
||||
]):
|
||||
r = repos[name] = make_repo(name)
|
||||
env['runbot_merge.repository'].create({
|
||||
'project_id': project.id,
|
||||
'name': r.name,
|
||||
'required_statuses': 'default',
|
||||
'group_id': False,
|
||||
**conf,
|
||||
})
|
||||
setreviewers(*project.repo_ids)
|
||||
env['runbot_merge.events_sources'].create([{'repository': r.name} for r in project.repo_ids])
|
||||
#endregion
|
||||
#region repos setup
|
||||
for repo_name, r in repos.items():
|
||||
with r:
|
||||
r.make_commits(
|
||||
None,
|
||||
Commit('initial', tree={
|
||||
'x': '1',
|
||||
f'{repo_name}/b': '2',
|
||||
f'{repo_name}/c': '3',
|
||||
}),
|
||||
ref='heads/master',
|
||||
)
|
||||
#endregion
|
||||
#region setup PR
|
||||
with (r := repos['b']):
|
||||
[c] = r.make_commits('master', Commit('second', tree={'b/c': '42'}), ref='heads/other')
|
||||
pr = r.make_pr(target='master', title='title', head='other')
|
||||
env.run_crons()
|
||||
with (r := repos['b']):
|
||||
pr.post_comment('hansen r+', config['role_reviewer']['token'])
|
||||
r.post_status(c, 'success')
|
||||
env.run_crons()
|
||||
# endregion
|
||||
|
||||
pr_id = to_pr(env, pr)
|
||||
assert not pr_id.blocked
|
||||
staging = env['runbot_merge.stagings'].search([])
|
||||
assert staging
|
||||
for r in repos.values():
|
||||
with r:
|
||||
r.post_status('staging.master', 'success')
|
||||
env.run_crons()
|
||||
assert staging.state == 'success'
|
||||
assert not staging.active
|
||||
assert staging.hash_pythonpath_layout
|
||||
|
||||
repo_pythonpath = copy.copy(repos['a'])
|
||||
repo_pythonpath.name = project.repo_pythonpath_layout
|
||||
c = repo_pythonpath.commit('master')
|
||||
t = repo_pythonpath.read_tree(c, recursive=True)
|
||||
del t['.gitmodules']
|
||||
del t['meta.json']
|
||||
|
||||
def name(repo_name):
|
||||
return repos[repo_name].name.split('/')[1]
|
||||
def ref(repo_name):
|
||||
return '@' + repos[repo_name].commit('master').id
|
||||
assert t == {
|
||||
name('a'): ref('a'),
|
||||
name('b'): ref('b'),
|
||||
'community/odoo/addons': f'../../../{name("b")}/b',
|
||||
f'{name("c")}/odoo/addons': ref('c'),
|
||||
f'.repos/{name("d")}': ref('d'),
|
||||
f'{name("d")}/odoo/upgrade': f'../../../.repos/{name("d")}/d',
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user