[ADD] runbot_merge: staging reifier

Add experimental support for creating submodule-based commits for
stagings (and batches), and pushing those in ancillary repositories.

Fixes #768

runbot_merge/__manifest__.py

@@ -10,6 +10,7 @@
         'models/crons/git_maintenance.xml',
         'models/crons/cleanup_scratch_branches.xml',
         'models/crons/issues_closer.xml',
+        'models/crons/staging_reifier.xml',
         'data/runbot_merge.pull_requests.feedback.template.csv',
         'views/res_partner.xml',
         'views/runbot_merge_project.xml',

runbot_merge/models/crons/__init__.py

@@ -1,3 +1,4 @@
 from . import git_maintenance
 from . import cleanup_scratch_branches
 from . import issues_closer
+from . import staging_reifier

runbot_merge/models/crons/staging_reifier.py

@@ -0,0 +1,350 @@
+""" Implements the reification of stagings into cross-repo git commits
+"""
+from __future__ import annotations
+
+import abc
+import datetime
+import itertools
+import json
+from dataclasses import dataclass
+from subprocess import PIPE
+from types import SimpleNamespace
+from typing import TypedDict
+
+from odoo import api, fields, models
+from ....runbot_merge import git
+from ..batch import Batch
+from ..pull_requests import Branch, StagingCommits
+
+
+class Project(models.Model):
+    _inherit = 'runbot_merge.project'
+
+    repo_flat_layout = fields.Char()
+    repo_pythonpath_layout = fields.Char()
+
+
+class Stagings(models.Model):
+    _inherit = 'runbot_merge.stagings'
+
+    # can not make these required because we need to have created the staging
+    # in order to know what its id is (also makes handling of batches easier)
+    # and NOT NULL can't be deferred
+    hash_flat_layout = fields.Char()
+    hash_pythonpath_layout = fields.Char()
+
+    id: int
+    target: Branch
+    batch_ids: Batch
+    commits: StagingCommits
+    staged_at: datetime.datetime
+
+
+class Repository(models.Model):
+    _inherit = 'runbot_merge.repository'
+
+    name: str
+    pythonpath_location = fields.Char(
+        help="Where the submodule should be located in the pythonpath layout, "
+             "if empty defaults to the root, unless `pythonpath_link_path` is "
+             "set, then defaults to `.repos`."
+             ""
+             "%(fullname)s, %(owner)s, and %(name)s are available as"
+             "placeholders in the path",
+    )
+    pythonpath_link_path = fields.Char(
+        help="Where the repository should be symlinked in the pythonpath layout,"
+             " leave empty to not symlink."
+             ""
+             "%(fullname)s, %(owner)s, and %(name)s are available as"
+             "placeholders in the path",
+    )
+    pythonpath_link_target = fields.Char(
+        help="Where the symlink should point to inside the repository, "
+             "leave empty for the root",
+    )
+
+class Reifier(models.Model):
+    _name = 'runbot_merge.staging.reifier'
+    _description = "transcriptor of staging into cross-repo git commits"
+    _order = 'create_date asc, id asc'
+
+    staging_id: Stagings = fields.Many2one('runbot_merge.stagings', required=True)
+    previous_staging_id: Stagings = fields.Many2one('runbot_merge.stagings')
+
+    @api.model_create_multi
+    def create(self, vals_list):
+        self.env.ref('runbot_merge.staging_reifier')._trigger()
+        return super().create(vals_list)
+
+    def _run(self):
+        projects = self.env['runbot_merge.project']
+        branches = set()
+        while r := self.search([], limit=1):
+            reify(r.staging_id, r.previous_staging_id)
+            projects |= r.staging_id.target.project_id
+            branches.add(r.staging_id.target)
+            r.unlink()
+            self.env.cr.commit()
+
+        for project in projects:
+            if name_flat := project.repo_flat_layout:
+                git.get_local(SimpleNamespace(
+                    name=name_flat,
+                    project_id=project,
+                )).push('origin', 'refs/heads/*:refs/heads/*')
+            if name_pythonpath := project.repo_pythonpath_layout:
+                git.get_local(SimpleNamespace(
+                    name=name_pythonpath,
+                    project_id=project,
+                )).push('origin', 'refs/heads/*:refs/heads/*')
+def reify(staging: Stagings, prev: Stagings) -> None:
+    project = staging.target.project_id
+    repos = project.repo_ids.having_branch(staging.target)
+
+    repo_flat = None
+    commit_flat = prev.hash_flat_layout
+    if name_flat := project.repo_flat_layout:
+        repo_flat = git.get_local(SimpleNamespace(
+            name=name_flat,
+            project_id=project,
+        )).with_config(check=True, encoding="utf-8", stdout=PIPE)
+
+    repo_pythonpath = None
+    commit_pythonpath = prev.hash_pythonpath_layout
+    if name_pythonpath := project.repo_pythonpath_layout:
+        repo_pythonpath = git.get_local(SimpleNamespace(
+            name=name_pythonpath,
+            project_id=project,
+        )).with_config(check=True, encoding="utf-8", stdout=PIPE)
+
+    if prev:
+        commits = {c.repository_id.name: c.commit_id.sha for c in prev.commits}
+    else:
+        # if there is no `prev` then we need to run the staging in reverse in
+        # order to get all the "root" commits for the staging, then we need to
+        # get their parents in order to know what state the repos were in
+        # before the staging, and *that* lets us figure out the per-batch state
+        commits = {c.repository_id.name: c.commit_id.sha for c in staging.commits}
+
+        # need to work off of the PR commits, because if no PR touches a
+        # repository then `commits` could hold *the first commit in the
+        # repository*, in which case we don't want to remove it
+        pr_commits = {}
+        for batch in reversed(staging.batch_ids):
+            # FIXME: broken if the PR has more than one commit and was rebased...
+            pr_commits.update(
+                (pr.repository.name, json.loads(pr.commits_map)[''])
+                for pr in batch.prs
+            )
+
+        for r, c in pr_commits.items():
+            repo = git.get_local(SimpleNamespace(
+                name=r,
+                project_id=project,
+            )).with_config(encoding="utf-8", stdout=PIPE)
+            if parent := repo.rev_parse('--revs-only', c + '~').stdout.strip():
+                commits[r] = parent
+            else:
+                # if a PR's commit has no parent then the PR itself introduced
+                # the repo (what?) in which case we don't have the repo at the
+                # staging and should ignore it
+                del commits[r]
+
+    for batch in staging.batch_ids:
+        commits.update(
+            (pr.repository.name, json.loads(pr.commits_map)[''])
+            for pr in batch.prs
+        )
+        prs = batch.prs.sorted('repository').mapped('display_name')
+        meta = json.dumps({
+            'staging': staging.id,
+            'batch': batch.id,
+            'label': batch.name,
+            'pull_requests': prs,
+            'commits': commits,
+        })
+
+        message = f"{staging.id}.{batch.id}: {batch.name}\n\n- " + "\n- ".join(prs)
+        if repo_flat:
+            tree_flat = layout_flat(
+                repo_flat, {'meta.json': GitBlob(meta)}, repos, commits)
+            commit_flat = repo_flat.commit_tree(
+                tree=tree_flat,
+                message=message,
+                parents=[commit_flat] if commit_flat else [],
+                author=(project.github_name, project.github_email, staging.staged_at.isoformat(timespec='seconds')),
+                committer=(project.github_name, project.github_email, staging.staged_at.isoformat(timespec='seconds')),
+            ).stdout.strip()
+        if repo_pythonpath:
+            tree_pythonpath = layout_pythonpath(
+                repo_pythonpath,
+                {'meta.json': GitBlob(meta)},
+                repos,
+                commits,
+            )
+            commit_pythonpath = repo_pythonpath.commit_tree(
+                tree=tree_pythonpath,
+                message=message,
+                parents=[commit_pythonpath] if commit_pythonpath else [],
+                author=('robodoo', 'robodoo@odoo.com', staging.staged_at.isoformat(timespec='seconds')),
+                committer=('robodoo', 'robodoo@odoo.com', staging.staged_at.isoformat(timespec='seconds')),
+            ).stdout.strip()
+
+    if repo_flat:
+        repo_flat.update_ref(
+            f'refs/heads/{staging.target.name}',
+            commit_flat,
+            prev.hash_flat_layout or '',
+        )
+    if repo_pythonpath:
+        repo_pythonpath.update_ref(
+            f'refs/heads/{staging.target.name}',
+            commit_pythonpath,
+            prev.hash_pythonpath_layout or '',
+        )
+    staging.write({
+        'hash_flat_layout': commit_flat,
+        'hash_pythonpath_layout': commit_pythonpath,
+    })
+
+
+def make_submodule(repo: str, path: str) -> str:
+    _, n = repo.split('/')
+    return f"""[submodule.{n}]
+    path = {path}
+    url = https://github.com/{repo}
+"""
+
+def layout_flat(
+    repo: git.Repo,
+    tree_init: dict[str, GitObject],
+    repos: Repository,
+    commits: dict[str, str],
+) -> str:
+    gitmodules = "\n".join(
+        make_submodule(repo.name, repo.name.split('/')[-1])
+        for repo in repos
+        if repo.name in commits
+    )
+
+    return GitTree({
+        **tree_init,
+        ".gitmodules": GitBlob(gitmodules),
+        **{
+            repo.name.split('/')[-1]: GitCommit(cc)
+            for repo in repos
+            if (cc := commits.get(repo.name))
+        }
+    }).write(repo, 0)
+
+def layout_pythonpath(
+    repo: git.Repo,
+    tree_init: dict[str, GitObject],
+    repos: Repository,
+    commits: dict[str, str],
+) -> str:
+    gitmodules = []
+    tree = GitTree(tree_init)
+    for repo_id in repos:
+        commit = commits.get(repo_id.name)
+        if commit is None:
+            continue
+
+        owner, name = repo_id.name.split('/')
+        tmpl = {'fullname': repo_id.name, 'owner': owner, 'name': name}
+        if p := repo_id.pythonpath_location:
+            path = p % tmpl
+        elif repo_id.pythonpath_link_path:
+            path = ".repos/%(name)s" % tmpl
+        else:
+            path = '%(name)s' % tmpl
+        link = (repo_id.pythonpath_link_path or '') % tmpl
+        target = repo_id.pythonpath_link_target or ''
+
+        tree.set(path, GitCommit(commit))
+        gitmodules.append(make_submodule(repo_id.name, path))
+
+        if link:
+            if target:
+                target = f"{path}/{target}"
+            else:
+                target = path
+            tree.set(link, GitLink(target))
+
+    tree.set(".gitmodules", GitBlob("\n".join(gitmodules)))
+    return tree.write(repo, 0)
+
+class Commit(TypedDict):
+    staging_id: int
+    commit_id: int
+    repository_id: int
+
+class GitObject(metaclass=abc.ABCMeta):
+    @property
+    @abc.abstractmethod
+    def mode(self) -> str:
+        ...
+
+    @property
+    @abc.abstractmethod
+    def type(self) -> str:
+        ...
+
+    @abc.abstractmethod
+    def write(self, repo: git.Repo, depth: int) -> str:
+        ...
+
+    def assert_tree(self) -> GitTree:
+        assert isinstance(self, GitTree)
+        return self
+
+@dataclass
+class GitCommit(GitObject):
+    sha: str
+    mode = "160000"
+    type = "commit"
+
+    def write(self, repo: git.Repo, _: int) -> str:
+        return self.sha
+
+@dataclass
+class GitTree(GitObject):
+    members: dict[str, GitObject]
+    mode = "40000"
+    type = "tree"
+
+    def set(self, path: str, obj: GitObject) -> None:
+        p, _, target = path.rpartition('/')
+        assert target, f"{path!r} can't be empty"
+        if p:
+            for part in p.split('/'):
+                self = self.members.setdefault(part, GitTree({})).assert_tree()
+        self.members[target] = obj
+
+    def write(self, repo: git.Repo, depth: int) -> str:
+        assert all(self.members), \
+            f"One of the tree entries has an empty filename {self.members}"
+        return repo.with_config(input="".join(
+            f"{obj.mode} {obj.type} {obj.write(repo, depth+1)}\t{name}\n"
+            for name, obj in self.members.items()
+        )).mktree().stdout.strip()
+
+@dataclass
+class GitLink(GitObject):
+    reference: str
+    mode = "120000"
+    type = "blob"
+
+    def write(self, repo: git.Repo, depth: int) -> str:
+        target = "/".join(itertools.repeat("..", depth)) + "/" + self.reference
+        return repo.with_config(input=target).hash_object("--stdin", "-w").stdout.strip()
+
+@dataclass
+class GitBlob(GitObject):
+    content: str
+    mode = "100644"
+    type = "blob"
+
+    def write(self, repo: git.Repo, _: int) -> str:
+        return repo.with_config(input=self.content).hash_object('--stdin', '-w').stdout.strip()

runbot_merge/models/crons/staging_reifier.xml

@@ -0,0 +1,23 @@
+<odoo>
+    <record id="access_staging_reifier" model="ir.model.access">
+        <field name="name">Access to staging reifier is useless</field>
+        <field name="model_id" ref="model_runbot_merge_staging_reifier"/>
+        <field name="perm_read">0</field>
+        <field name="perm_create">0</field>
+        <field name="perm_write">0</field>
+        <field name="perm_unlink">0</field>
+    </record>
+
+    <record model="ir.cron" id="staging_reifier">
+        <field name="name">Reifiy stagings to cross-repo commits</field>
+        <field name="model_id" ref="model_runbot_merge_staging_reifier"/>
+        <field name="state">code</field>
+        <field name="code">model._run()</field>
+        <!--
+        nota: even though this is only triggered, numbercall has to be
+        non-zero because the counter is taken in account by cron triggers
+        -->
+        <field name="numbercall">-1</field>
+        <field name="doall" eval="False"/>
+    </record>
+</odoo>

runbot_merge/models/pull_requests.py

@@ -2435,6 +2435,37 @@ class Stagings(models.Model):
                         })
                 if self.issues_to_close:
                     self.env['runbot_merge.issues_closer'].create(self.issues_to_close)
+
+                # FIXME: error prone, should probably store the previous
+                #        staging on creation instead?
+                last2 = self.with_context(active_search=False).search([
+                    ('state', '=', 'success'),
+                    ('target', '=', self.target.id),
+                ], order='id desc', limit=2)
+                if len(last2) == 2:
+                    _self, previous = last2
+                elif self.target != project.branch_ids[:1]:
+                    _self = last2
+                    previous = self.search([
+                        ('state', '=', 'success'),
+                        ('target', '=', project.branch_ids[:1].id),
+                        ('active', '=', False),
+                    ], order='id desc', limit=1)
+                else:
+                    # no previous branch
+                    _self = last2
+                    previous = self.browse()
+                if self == _self:
+                    self.env['runbot_merge.staging.reifier'].create({
+                        'previous_staging_id': previous.id,
+                        'staging_id': self.id,
+                    })
+                else:
+                    _logger.warning(
+                        "Got different self (%s) and last success (%s)",
+                        self,
+                        _self,
+                    )
             finally:
                 self.write({'active': False})
         elif self.state == 'failure' or self.is_timed_out():

runbot_merge/tests/test_staging_reifier.py

@@ -0,0 +1,95 @@
+import copy
+import pprint
+
+import pytest
+
+from utils import Commit, to_pr
+
+
+def test_basic(make_repo, project, env, setreviewers, config, users, partners, pytestconfig):
+    repos = {}
+    #region project setup
+    project.repo_pythonpath_layout = make_repo('pythonpath', hooks=False).name
+    for name, conf in zip('abcd', [
+        # flat-style repo (odoo/documentation)
+        {},
+        # flat repo with symlink into (odoo/odoo)
+        {
+            'pythonpath_location': '%(name)s',
+            'pythonpath_link_path': 'community/odoo/addons',
+            'pythonpath_link_target': 'b',
+        },
+        # modules directory style
+        {'pythonpath_location': '%(name)s/odoo/addons'},
+        # upgrade style
+        {
+            'pythonpath_link_path': '%(name)s/odoo/upgrade',
+            'pythonpath_link_target': 'd',
+        },
+    ]):
+        r = repos[name] = make_repo(name)
+        env['runbot_merge.repository'].create({
+            'project_id': project.id,
+            'name': r.name,
+            'required_statuses': 'default',
+            'group_id': False,
+            **conf,
+        })
+    setreviewers(*project.repo_ids)
+    env['runbot_merge.events_sources'].create([{'repository': r.name} for r in project.repo_ids])
+    #endregion
+    #region repos setup
+    for repo_name, r in repos.items():
+        with r:
+            r.make_commits(
+                None,
+                Commit('initial', tree={
+                    'x': '1',
+                    f'{repo_name}/b': '2',
+                    f'{repo_name}/c': '3',
+                }),
+                ref='heads/master',
+            )
+    #endregion
+    #region setup PR
+    with (r := repos['b']):
+        [c] = r.make_commits('master', Commit('second', tree={'b/c': '42'}), ref='heads/other')
+        pr = r.make_pr(target='master', title='title', head='other')
+    env.run_crons()
+    with (r := repos['b']):
+        pr.post_comment('hansen r+', config['role_reviewer']['token'])
+        r.post_status(c, 'success')
+    env.run_crons()
+    # endregion
+
+    pr_id = to_pr(env, pr)
+    assert not pr_id.blocked
+    staging = env['runbot_merge.stagings'].search([])
+    assert staging
+    for r in repos.values():
+        with r:
+            r.post_status('staging.master', 'success')
+    env.run_crons()
+    assert staging.state == 'success'
+    assert not staging.active
+    assert staging.hash_pythonpath_layout
+
+    repo_pythonpath = copy.copy(repos['a'])
+    repo_pythonpath.name = project.repo_pythonpath_layout
+    c = repo_pythonpath.commit('master')
+    t = repo_pythonpath.read_tree(c, recursive=True)
+    del t['.gitmodules']
+    del t['meta.json']
+
+    def name(repo_name):
+        return repos[repo_name].name.split('/')[1]
+    def ref(repo_name):
+        return '@' + repos[repo_name].commit('master').id
+    assert t == {
+        name('a'): ref('a'),
+        name('b'): ref('b'),
+        'community/odoo/addons': f'../../../{name("b")}/b',
+        f'{name("c")}/odoo/addons': ref('c'),
+        f'.repos/{name("d")}': ref('d'),
+        f'{name("d")}/odoo/upgrade': f'../../../.repos/{name("d")}/d',
+    }