From 4f237d15b077da08b9ab1b8b174a4b37381f6cb4 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 7 Jun 2023 09:19:48 +0200
Subject: [PATCH 001/192] [FIX] runbot_merge: correctly check request in
 handle_pr

652b1ff9aea395b2bc24004e0e0741631190ba26 wanted to check if a request
was available, however it deref'd the `request` object without
checking it which is not correct: a `request` normally has an
`httprequest`, but the `request` itself might be missing if the
handler is called from e.g. a cron.

Fixes #739
---
 runbot_merge/controllers/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 87b84606..efb752ac 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -154,7 +154,7 @@ def handle_pr(env, event):
     if not branch:
         return "Not set up to care about {}:{}".format(r, b)
 
-    headers = request.httprequest.headers if request.httprequest else {}
+    headers = request.httprequest.headers if request else {}
     _logger.info(
         "%s: %s#%s (%s) (by %s, delivery %s by %s)",
         event['action'],

From 485d2d7b55ed3c909eebad0d0dbf0690d60b2acd Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 7 Jun 2023 09:54:06 +0200
Subject: [PATCH 002/192] [IMP] runbot_merge: add sitemap params to http
 controllers

When it's missing, website complains because it's dumb.

Fixes #763
---
 runbot_merge/controllers/dashboard.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/runbot_merge/controllers/dashboard.py b/runbot_merge/controllers/dashboard.py
index e80a2f15..f15c52d2 100644
--- a/runbot_merge/controllers/dashboard.py
+++ b/runbot_merge/controllers/dashboard.py
@@ -11,13 +11,13 @@ from odoo.http import Controller, route, request
 
 LIMIT = 20
 class MergebotDashboard(Controller):
-    @route('/runbot_merge', auth="public", type="http", website=True)
+    @route('/runbot_merge', auth="public", type="http", website=True, sitemap=True)
     def dashboard(self):
         return request.render('runbot_merge.dashboard', {
             'projects': request.env['runbot_merge.project'].with_context(active_test=False).sudo().search([]),
         })
 
-    @route('/runbot_merge/<int:branch_id>', auth='public', type='http', website=True)
+    @route('/runbot_merge/<int:branch_id>', auth='public', type='http', website=True, sitemap=False)
     def stagings(self, branch_id, until=None):
         branch = request.env['runbot_merge.branch'].browse(branch_id).sudo().exists()
         if not branch:
@@ -49,7 +49,7 @@ class MergebotDashboard(Controller):
             entries.setdefault(key, []).extend(map(item_converter, items))
         return entries
 
-    @route('/runbot_merge/changelog', auth='public', type='http', website=True)
+    @route('/runbot_merge/changelog', auth='public', type='http', website=True, sitemap=True)
     def changelog(self):
         md = markdown.Markdown(extensions=['nl2br'], output_format='html5')
         entries = self.entries(lambda t: markupsafe.Markup(md.convert(t)))
@@ -57,7 +57,7 @@ class MergebotDashboard(Controller):
             'entries': entries,
         })
 
-    @route('/<org>/<repo>/pull/<int(min=1):pr>', auth='public', type='http', website=True)
+    @route('/<org>/<repo>/pull/<int(min=1):pr>', auth='public', type='http', website=True, sitemap=False)
     def pr(self, org, repo, pr):
         pr_id = request.env['runbot_merge.pull_requests'].sudo().search([
             ('repository.name', '=', f'{org}/{repo}'),

From 611f9150ff1d9e2bcdf8514642f15fecdf9ae464 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 7 Jun 2023 11:14:47 +0200
Subject: [PATCH 003/192] [IMP] runbot_merge: add signed kw support to
 from_role, use it

Closes #774
---
 conftest.py                                       |  2 +-
 runbot_merge/controllers/reviewer_provisioning.py | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/conftest.py b/conftest.py
index c47c07a0..09ca14b7 100644
--- a/conftest.py
+++ b/conftest.py
@@ -355,7 +355,7 @@ def dummy_addons_path():
             'version': '1.0',
         }), encoding='utf-8')
         (mod / 'util.py').write_text("""\
-def from_role(_):
+def from_role(*_, **__):
     return lambda fn: fn
 """, encoding='utf-8')
 
diff --git a/runbot_merge/controllers/reviewer_provisioning.py b/runbot_merge/controllers/reviewer_provisioning.py
index 1b7a9f2f..6a9b3af4 100644
--- a/runbot_merge/controllers/reviewer_provisioning.py
+++ b/runbot_merge/controllers/reviewer_provisioning.py
@@ -6,12 +6,12 @@ from odoo.http import Controller, request, route
 try:
     from odoo.addons.saas_worker.util import from_role
 except ImportError:
-    def from_role(_):
+    def from_role(*_, **__):
         return lambda _: None
 
 _logger = logging.getLogger(__name__)
 class MergebotReviewerProvisioning(Controller):
-    @from_role('accounts')
+    @from_role('accounts', signed=True)
     @route('/runbot_merge/users', type='json', auth='public')
     def list_users(self):
         env = request.env(su=True)
@@ -23,7 +23,7 @@ class MergebotReviewerProvisioning(Controller):
             if u.github_login
         ]
 
-    @from_role('accounts')
+    @from_role('accounts', signed=True)
     @route('/runbot_merge/provision', type='json', auth='public')
     def provision_user(self, users):
         _logger.info('Provisioning %s users: %s.', len(users), ', '.join(map(
@@ -107,7 +107,7 @@ class MergebotReviewerProvisioning(Controller):
         _logger.info("Provisioning: created %d updated %d.", created, updated)
         return [created, updated]
 
-    @from_role('accounts')
+    @from_role('accounts', signed=True)
     @route(['/runbot_merge/get_reviewers'], type='json', auth='public')
     def fetch_reviewers(self, **kwargs):
         reviewers = request.env['res.partner.review'].sudo().search([
@@ -115,7 +115,7 @@ class MergebotReviewerProvisioning(Controller):
         ]).mapped('partner_id.github_login')
         return reviewers
 
-    @from_role('accounts')
+    @from_role('accounts', signed=True)
     @route(['/runbot_merge/remove_reviewers'], type='json', auth='public', methods=['POST'])
     def update_reviewers(self, github_logins, **kwargs):
         partners = request.env['res.partner'].sudo().search([('github_login', 'in', github_logins)])

From 4a4252b4b912158365e3afa56aeb18d9115dd3e3 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 7 Jun 2023 14:42:08 +0200
Subject: [PATCH 004/192] [FIX] runbot_merge: holes in provisioning

- github logins are case-insensitive while the db field is CI the dict
  in which partners are stored for matching is not, And the caller may
  not preserve casing.

  Thus it's necessary to check the casefolded database values against
  casefolded parameters, rather than exactly.
- users may get disabled by mistake or when one leaves the project,
  they may also get switched from internal to portal, therefore it is
  necessary to re-enable and re-enroll them if they come back.
- while at it remove the user's email when they depart, as they likely
  use an organisational email which they don't have access to anymore

Side-note, but remove the limit on the number of users / partners
being created at once: because there are almost no modules in the
mergebot's instance, creating partner goes quite fast (compared to a
full instance), thus the limitation is almost certainly unnecessary
(creating ~300 users seems to take ~450ms).

Fixes ##776
---
 conftest.py                                   |  9 ++
 .../controllers/reviewer_provisioning.py      | 38 +++++---
 runbot_merge/tests/test_provisioning.py       | 97 +++++++++++++++++--
 3 files changed, 127 insertions(+), 17 deletions(-)

diff --git a/conftest.py b/conftest.py
index 09ca14b7..8e7b05f9 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1078,6 +1078,15 @@ class Environment:
     def __getitem__(self, name):
         return Model(self, name)
 
+    def ref(self, xid, raise_if_not_found=True):
+        model, obj_id = self(
+            'ir.model.data', 'check_object_reference',
+            *xid.split('.', 1),
+            raise_on_access_error=raise_if_not_found
+        )
+        return Model(self, model, [obj_id]) if obj_id else None
+
+
     def run_crons(self, *xids, **kw):
         crons = xids or self._default_crons
         print('running crons', crons, file=sys.stderr)
diff --git a/runbot_merge/controllers/reviewer_provisioning.py b/runbot_merge/controllers/reviewer_provisioning.py
index 6a9b3af4..e0aa0fc7 100644
--- a/runbot_merge/controllers/reviewer_provisioning.py
+++ b/runbot_merge/controllers/reviewer_provisioning.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import logging
 
+from odoo import Command
 from odoo.http import Controller, request, route
 
 try:
@@ -34,7 +35,7 @@ class MergebotReviewerProvisioning(Controller):
         Partners = env['res.partner']
         Users = env['res.users']
 
-        existing_partners = Partners.search([
+        existing_partners = Partners.with_context(active_test=False).search([
             '|', ('email', 'in', [u['email'] for u in users]),
                  ('github_login', 'in', [u['github_login'] for u in users])
         ])
@@ -55,20 +56,24 @@ class MergebotReviewerProvisioning(Controller):
             if p.github_login:
                 # assume there can't be an existing one because github_login is
                 # unique, and should not be able to collide with emails
-                partners[p.github_login] = p
+                partners[p.github_login.casefold()] = p
 
+        portal = env.ref('base.group_portal')
         internal = env.ref('base.group_user')
         odoo_provider = env.ref('auth_oauth.provider_openerp')
 
         to_create = []
-        created = updated = 0
+        updated = 0
+        to_activate = Partners
         for new in users:
             if 'sub' in new:
                 new['oauth_provider_id'] = odoo_provider.id
                 new['oauth_uid'] = new.pop('sub')
 
             # prioritise by github_login as that's the unique-est point of information
-            current = partners.get(new['github_login']) or partners.get(new['email']) or Partners
+            current = partners.get(new['github_login'].casefold()) or partners.get(new['email']) or Partners
+            if not current.active:
+                to_activate |= current
             # entry doesn't have user -> create user
             if not current.user_ids:
                 # skip users without an email (= login) as that
@@ -77,7 +82,7 @@ class MergebotReviewerProvisioning(Controller):
                     continue
 
                 new['login'] = new['email']
-                new['groups_id'] = [(4, internal.id)]
+                new['groups_id'] = [Command.link(internal.id)]
                 # entry has partner -> create user linked to existing partner
                 # (and update partner implicitly)
                 if current:
@@ -90,19 +95,29 @@ class MergebotReviewerProvisioning(Controller):
             if len(user) != 1:
                 _logger.warning("Got %d users for partner %s.", len(user), current.display_name)
                 user = user[:1]
+            new.setdefault("active", True)
             update_vals = {
                 k: v
                 for k, v in new.items()
-                if v not in ('login', 'email')
                 if v != (user[k] if k != 'oauth_provider_id' else user[k].id)
             }
+            if user.has_group('base.group_portal'):
+                update_vals['groups_id'] = [
+                    Command.unlink(portal.id),
+                    Command.link(internal.id),
+                ]
+
             if update_vals:
                 user.write(update_vals)
                 updated += 1
+
+        created = len(to_create)
         if to_create:
             # only create 100 users at a time to avoid request timeout
-            Users.create(to_create[:100])
-            created = len(to_create[:100])
+            Users.create(to_create)
+
+        if to_activate:
+            to_activate.active = True
 
         _logger.info("Provisioning: created %d updated %d.", created, updated)
         return [created, updated]
@@ -120,12 +135,13 @@ class MergebotReviewerProvisioning(Controller):
     def update_reviewers(self, github_logins, **kwargs):
         partners = request.env['res.partner'].sudo().search([('github_login', 'in', github_logins)])
         partners.write({
-            'review_rights': [(5, 0, 0)],
-            'delegate_reviewer': [(5, 0, 0)],
+            'email': False,
+            'review_rights': [Command.clear()],
+            'delegate_reviewer': [Command.clear()],
         })
 
         # Assign the linked users as portal users
         partners.mapped('user_ids').write({
-            'groups_id': [(6, 0, [request.env.ref('base.group_portal').id])]
+            'groups_id': [Command.set([request.env.ref('base.group_portal').id])]
         })
         return True
diff --git a/runbot_merge/tests/test_provisioning.py b/runbot_merge/tests/test_provisioning.py
index 6119bf17..af939305 100644
--- a/runbot_merge/tests/test_provisioning.py
+++ b/runbot_merge/tests/test_provisioning.py
@@ -15,10 +15,8 @@ def test_basic_provisioning(env, port):
     assert g.partner_id.name == GEORGE['name']
     assert g.partner_id.github_login == GEORGE['github_login']
     assert g.oauth_uid == GEORGE['sub']
-    (model, g_id) = env['ir.model.data']\
-        .check_object_reference('base', 'group_user')
-    assert model == 'res.groups'
-    assert g.groups_id.id == g_id, "check that users were provisioned as internal (not portal)"
+    internal = env.ref('base.group_user')
+    assert (g.groups_id & internal) == internal, "check that users were provisioned as internal (not portal)"
 
     # repeated provisioning should be a no-op
     r = provision_user(port, [GEORGE])
@@ -79,8 +77,20 @@ def test_upgrade_partner(env, port):
         'email': GEORGE['email'],
     }]
 
-    p.user_ids.unlink()
-    p.unlink()
+    # deactivate user to see if provisioning re-enables them
+    p.user_ids.active = False
+    r = provision_user(port, [GEORGE])
+    assert r == [0, 1]
+    assert len(p.user_ids) == 1, "provisioning should re-enable user"
+    assert p.user_ids.active
+
+    # re-enable both user and partner to see if both get re-enabled
+    p.user_ids.active = False
+    p.active = False
+    r = provision_user(port, [GEORGE])
+    assert r == [0, 1]
+    assert p.active, "provisioning should re-enable partner"
+    assert p.user_ids.active
 
 def test_no_email(env, port):
     """ Provisioning system should ignore email-less entries
@@ -88,6 +98,81 @@ def test_no_email(env, port):
     r = provision_user(port, [{**GEORGE, 'email': None}])
     assert r == [0, 0]
 
+def test_casing(env, port):
+    p = env['res.partner'].create({
+        'name': 'Bob',
+        'github_login': "Bob",
+    })
+    assert not p.user_ids
+    assert provision_user(port, [{
+        'name': "Bob Thebuilder",
+        'github_login': "bob",
+        'email': 'bob@example.org',
+        'sub': '5473634',
+    }]) == [1, 0]
+
+    assert p.user_ids.name == 'Bob Thebuilder'
+    assert p.user_ids.email == 'bob@example.org'
+    assert p.user_ids.oauth_uid == '5473634'
+    # should be written on the partner through the user
+    assert p.name == 'Bob Thebuilder'
+    assert p.email == 'bob@example.org'
+    assert p.github_login == 'bob'
+
+def test_user_leaves_and_returns(env, port):
+    internal = env.ref('base.group_user')
+    portal = env.ref('base.group_portal')
+    categories = internal | portal | env.ref('base.group_public')
+
+    assert provision_user(port, [{
+        "name": "Bamien Douvy",
+        "github_login": "DouvyB",
+        "email": "bado@example.org",
+        "sub": "123456",
+    }]) == [1, 0]
+    p = env['res.partner'].search([('github_login', '=', "DouvyB")])
+    assert (p.user_ids.groups_id & categories) == internal
+
+    # bye bye 👋
+    requests.post(f'http://localhost:{port}/runbot_merge/remove_reviewers', json={
+        'jsonrpc': '2.0',
+        'id': None,
+        'method': 'call',
+        'params': {'github_logins': ['douvyb']},
+    })
+    assert (p.user_ids.groups_id & categories) == portal
+    assert p.email is False
+
+    # he's back ❤️
+    assert provision_user(port, [{
+        "name": "Bamien Douvy",
+        "github_login": "DouvyB",
+        "email": "bado@example.org",
+        "sub": "123456",
+    }]) == [0, 1]
+    assert (p.user_ids.groups_id & categories) == internal
+    assert p.email == 'bado@example.org'
+
+def test_bulk_ops(env, port):
+    a, b = env['res.partner'].create([{
+        'name': "Bob",
+        'email': "bob@example.org",
+        'active': False,
+    }, {
+        'name': "Coc",
+        'email': "coc@example.org",
+        'active': False,
+    }])
+    assert a.active == b.active == False
+
+    assert provision_user(port, [
+        {'email': 'bob@example.org', 'github_login': 'xyz'},
+        {'email': 'coc@example.org', 'github_login': 'abc'},
+    ]) == [2, 0]
+    assert a.users_id
+    assert b.users_id
+    assert a.active == b.active == True
+
 def provision_user(port, users):
     r = requests.post(f'http://localhost:{port}/runbot_merge/provision', json={
         'jsonrpc': '2.0',

From 2009177adae5b8aec600c0a6d9bf916666cf3504 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 8 Jun 2023 08:19:43 +0200
Subject: [PATCH 005/192] [IMP] *: allow disabling staging on branch, remove fp
 target flag

- currently disabling staging only works globally, allow disabling on
  a single branch

  - use a toggle
  - remove a pair of tests which work specifically with `fp_target`,
    can't work with `active` (probably)
  - cleanup search of possible and active stagings, add relevant
    indexes and use direct search of relevant branches instead of
    looking up from the project

- also use toggle button for `active` on branches
- shitty workaround for upgrading DB: apparently mail really wants to
  have a `user_id` to do some weird thing, so need to re-add it after
  resetting everything

Fixes #727
---
 forwardport/data/views.xml                  |  6 --
 forwardport/models/project.py               | 15 +---
 forwardport/tests/test_conflicts.py         |  2 +-
 forwardport/tests/test_limit.py             | 77 +--------------------
 forwardport/tests/test_updates.py           | 12 ++--
 forwardport/tests/test_weird.py             | 16 ++---
 mergebot_test_utils/utils.py                |  6 +-
 runbot_merge/models/project.py              | 31 +++++----
 runbot_merge/models/pull_requests.py        |  8 ++-
 runbot_merge/tests/test_staging.py          | 41 +++++++++++
 runbot_merge/views/res_partner.xml          |  1 +
 runbot_merge/views/runbot_merge_project.xml |  3 +-
 12 files changed, 88 insertions(+), 130 deletions(-)
 create mode 100644 runbot_merge/tests/test_staging.py

diff --git a/forwardport/data/views.xml b/forwardport/data/views.xml
index 46a8b48a..c25bb311 100644
--- a/forwardport/data/views.xml
+++ b/forwardport/data/views.xml
@@ -152,12 +152,6 @@
                        help="Repository where forward port branches will be created"
                 />
             </xpath>
-
-            <xpath expr="//field[@name='branch_ids']/tree" position="inside">
-                <field name="fp_target" string="FP to"
-                       help="This branch will be forward-ported to (from lower ones)"
-                />
-            </xpath>
         </field>
     </record>
 
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index f59bdbbd..fcf7c645 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -210,17 +210,6 @@ class Repository(models.Model):
     _inherit = 'runbot_merge.repository'
     fp_remote_target = fields.Char(help="where FP branches get pushed")
 
-class Branch(models.Model):
-    _inherit = 'runbot_merge.branch'
-
-    fp_target = fields.Boolean(default=True)
-    fp_enabled = fields.Boolean(compute='_compute_fp_enabled')
-
-    @api.depends('active', 'fp_target')
-    def _compute_fp_enabled(self):
-        for b in self:
-            b.fp_enabled = b.active and b.fp_target
-
 class PullRequests(models.Model):
     _inherit = 'runbot_merge.pull_requests'
 
@@ -438,7 +427,7 @@ class PullRequests(models.Model):
                         ping = False
                         msg = "Forward-port disabled."
                         self.limit_id = limit_id
-                    elif not limit_id.fp_enabled:
+                    elif not limit_id.active:
                         msg = "branch %r is disabled, it can't be used as a forward port target." % limit_id.name
                     else:
                         ping = False
@@ -550,7 +539,7 @@ class PullRequests(models.Model):
         return next((
             branch
             for branch in branches[from_+1:to_+1]
-            if branch.fp_enabled
+            if branch.active
         ), None)
 
     def _commits_lazy(self):
diff --git a/forwardport/tests/test_conflicts.py b/forwardport/tests/test_conflicts.py
index 4e6dd084..3cda8430 100644
--- a/forwardport/tests/test_conflicts.py
+++ b/forwardport/tests/test_conflicts.py
@@ -16,7 +16,7 @@ def test_conflict(env, config, make_repo, users):
     project = env['runbot_merge.project'].search([])
     project.write({
         'branch_ids': [
-            (0, 0, {'name': 'd', 'sequence': 40, 'fp_target': True})
+            (0, 0, {'name': 'd', 'sequence': 40})
         ]
     })
 
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index 8310619d..fca04111 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -50,31 +50,6 @@ def test_configure(env, config, make_repo):
     assert prs[2].number == originals[2]
 
 
-def test_self_disabled(env, config, make_repo):
-    """ Allow setting target as limit even if it's disabled
-    """
-    prod, other = make_basic(env, config, make_repo)
-    bot_name = env['runbot_merge.project'].search([]).fp_github_name
-    branch_a = env['runbot_merge.branch'].search([('name', '=', 'a')])
-    branch_a.fp_target = False
-    with prod:
-        [c] = prod.make_commits('a', Commit('c', tree={'0': '0'}), ref='heads/mybranch')
-        pr = prod.make_pr(target='a', head='mybranch')
-        prod.post_status(c, 'success', 'legal/cla')
-        prod.post_status(c, 'success', 'ci/runbot')
-        pr.post_comment('hansen r+\n%s up to a' % bot_name, config['role_reviewer']['token'])
-    env.run_crons()
-    pr_id = env['runbot_merge.pull_requests'].search([('number', '=', pr.number)])
-    assert pr_id.limit_id == branch_a
-
-    with prod:
-        prod.post_status('staging.a', 'success', 'legal/cla')
-        prod.post_status('staging.a', 'success', 'ci/runbot')
-
-    assert env['runbot_merge.pull_requests'].search([]) == pr_id,\
-        "should not have created a forward port"
-
-
 def test_ignore(env, config, make_repo):
     """ Provide an "ignore" command which is equivalent to setting the limit
     to target
@@ -100,17 +75,12 @@ def test_ignore(env, config, make_repo):
         "should not have created a forward port"
 
 
-@pytest.mark.parametrize('enabled', ['active', 'fp_target'])
-def test_disable(env, config, make_repo, users, enabled):
+def test_disable(env, config, make_repo, users):
     """ Checks behaviour if the limit target is disabled:
 
     * disable target while FP is ongoing -> skip over (and stop there so no FP)
     * forward-port over a disabled branch
     * request a disabled target as limit
-
-    Disabling (with respect to forward ports) can be performed by marking the
-    branch as !active (which also affects mergebot operations), or as
-    !fp_target (won't be forward-ported to).
     """
     prod, other = make_basic(env, config, make_repo)
     project = env['runbot_merge.project'].search([])
@@ -133,7 +103,7 @@ def test_disable(env, config, make_repo, users, enabled):
         prod.post_status('staging.a', 'success', 'legal/cla')
         prod.post_status('staging.a', 'success', 'ci/runbot')
     # disable branch b
-    env['runbot_merge.branch'].search([('name', '=', 'b')]).write({enabled: False})
+    env['runbot_merge.branch'].search([('name', '=', 'b')]).active = False
     env.run_crons()
 
     # should have created a single PR (to branch c, for pr 1)
@@ -168,49 +138,6 @@ def test_disable(env, config, make_repo, users, enabled):
         seen(env, pr, users),
     }
 
-
-def test_default_disabled(env, config, make_repo, users):
-    """ If the default limit is disabled, it should still be the default
-    limit but the ping message should be set on the actual last FP (to the
-    last non-deactivated target)
-    """
-    prod, other = make_basic(env, config, make_repo)
-    branch_c = env['runbot_merge.branch'].search([('name', '=', 'c')])
-    branch_c.fp_target = False
-
-    with prod:
-        [c] = prod.make_commits('a', Commit('c', tree={'0': '0'}), ref='heads/branch0')
-        pr = prod.make_pr(target='a', head='branch0')
-        prod.post_status(c, 'success', 'legal/cla')
-        prod.post_status(c, 'success', 'ci/runbot')
-        pr.post_comment('hansen r+', config['role_reviewer']['token'])
-    env.run_crons()
-
-    assert env['runbot_merge.pull_requests'].search([]).limit_id == branch_c
-
-    with prod:
-        prod.post_status('staging.a', 'success', 'legal/cla')
-        prod.post_status('staging.a', 'success', 'ci/runbot')
-    env.run_crons()
-
-    p1, p2 = env['runbot_merge.pull_requests'].search([], order='number')
-    assert p1.number == pr.number
-    pr2 = prod.get_pr(p2.number)
-
-    cs = pr2.comments
-    assert len(cs) == 2
-    assert pr2.comments == [
-        seen(env, pr2, users),
-        (users['user'], """\
-@%(user)s @%(reviewer)s this PR targets b and is the last of the forward-port chain.
-
-To merge the full chain, say
-> @%(user)s r+
-
-More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
-""" % users)
-    ]
-
 def test_limit_after_merge(env, config, make_repo, users):
     """ If attempting to set a limit (<up to>) on a PR which is merged
     (already forward-ported or not), or is a forward-port PR, fwbot should
diff --git a/forwardport/tests/test_updates.py b/forwardport/tests/test_updates.py
index 99471cd7..384df08f 100644
--- a/forwardport/tests/test_updates.py
+++ b/forwardport/tests/test_updates.py
@@ -151,9 +151,7 @@ def test_update_merged(env, make_repo, config, users):
     with prod:
         prod.make_ref('heads/d', prod.commit('c').id)
     env['runbot_merge.project'].search([]).write({
-        'branch_ids': [(0, 0, {
-            'name': 'd', 'sequence': 40, 'fp_target': True,
-        })]
+        'branch_ids': [(0, 0, {'name': 'd', 'sequence': 40})]
     })
 
     with prod:
@@ -251,10 +249,10 @@ def test_duplicate_fw(env, make_repo, setreviewers, config, users):
         'github_prefix': 'hansen',
         'fp_github_token': config['github']['token'],
         'branch_ids': [
-            (0, 0, {'name': 'master', 'sequence': 0, 'fp_target': True}),
-            (0, 0, {'name': 'v3', 'sequence': 1, 'fp_target': True}),
-            (0, 0, {'name': 'v2', 'sequence': 2, 'fp_target': True}),
-            (0, 0, {'name': 'v1', 'sequence': 3, 'fp_target': True}),
+            (0, 0, {'name': 'master', 'sequence': 0}),
+            (0, 0, {'name': 'v3', 'sequence': 1}),
+            (0, 0, {'name': 'v2', 'sequence': 2}),
+            (0, 0, {'name': 'v1', 'sequence': 3}),
         ],
         'repo_ids': [
             (0, 0, {
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index ab077c05..4ef56feb 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -26,9 +26,9 @@ def make_basic(env, config, make_repo, *, fp_token, fp_remote):
             'github_prefix': 'hansen',
             'fp_github_token': fp_token and config['github']['token'],
             'branch_ids': [
-                (0, 0, {'name': 'a', 'sequence': 2, 'fp_target': True}),
-                (0, 0, {'name': 'b', 'sequence': 1, 'fp_target': True}),
-                (0, 0, {'name': 'c', 'sequence': 0, 'fp_target': True}),
+                (0, 0, {'name': 'a', 'sequence': 2}),
+                (0, 0, {'name': 'b', 'sequence': 1}),
+                (0, 0, {'name': 'c', 'sequence': 0}),
             ],
         })
 
@@ -263,9 +263,9 @@ class TestNotAllBranches:
             'github_prefix': 'hansen',
             'fp_github_token': config['github']['token'],
             'branch_ids': [
-                (0, 0, {'name': 'a', 'sequence': 2, 'fp_target': True}),
-                (0, 0, {'name': 'b', 'sequence': 1, 'fp_target': True}),
-                (0, 0, {'name': 'c', 'sequence': 0, 'fp_target': True}),
+                (0, 0, {'name': 'a', 'sequence': 2}),
+                (0, 0, {'name': 'b', 'sequence': 1}),
+                (0, 0, {'name': 'c', 'sequence': 0}),
             ]
         })
         repo_a = env['runbot_merge.repository'].create({
@@ -514,7 +514,7 @@ def test_new_intermediate_branch(env, config, make_repo):
     env.run_crons()
     project.write({
         'branch_ids': [
-            (0, False, {'name': 'new', 'sequence': 1, 'fp_target': True}),
+            (0, False, {'name': 'new', 'sequence': 1}),
         ]
     })
     env.run_crons()
@@ -748,7 +748,7 @@ def test_retarget_after_freeze(env, config, make_repo, users):
     project.write({
         'branch_ids': [
             (1, branch_c.id, {'sequence': 1}),
-            (0, 0, {'name': 'bprime', 'sequence': 2, 'fp_target': True}),
+            (0, 0, {'name': 'bprime', 'sequence': 2}),
             (1, branch_b.id, {'sequence': 3}),
             (1, branch_a.id, {'sequence': 4}),
         ]
diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index 2df8995d..a23d6726 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -74,9 +74,9 @@ def make_basic(env, config, make_repo, *, reponame='proj', project_name='myproje
             'github_prefix': 'hansen',
             'fp_github_token': config['github']['token'],
             'branch_ids': [
-                (0, 0, {'name': 'a', 'sequence': 100, 'fp_target': True}),
-                (0, 0, {'name': 'b', 'sequence': 80, 'fp_target': True}),
-                (0, 0, {'name': 'c', 'sequence': 60, 'fp_target': True}),
+                (0, 0, {'name': 'a', 'sequence': 100}),
+                (0, 0, {'name': 'b', 'sequence': 80}),
+                (0, 0, {'name': 'c', 'sequence': 60}),
             ],
         })
 
diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index fb46c33f..883c91c6 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -46,10 +46,11 @@ class Project(models.Model):
     freeze_reminder = fields.Text()
 
     def _check_stagings(self, commit=False):
-        for branch in self.search([]).mapped('branch_ids').filtered('active'):
+        # check branches with an active staging
+        for branch in self.env['runbot_merge.branch']\
+                .with_context(active_test=False)\
+                .search([('active_staging_id', '!=', False)]):
             staging = branch.active_staging_id
-            if not staging:
-                continue
             try:
                 with self.env.cr.savepoint():
                     staging.check_status()
@@ -61,16 +62,20 @@ class Project(models.Model):
                     self.env.cr.commit()
 
     def _create_stagings(self, commit=False):
-        for branch in self.search([]).mapped('branch_ids').filtered('active'):
-            if not branch.active_staging_id:
-                try:
-                    with self.env.cr.savepoint():
-                        branch.try_staging()
-                except Exception:
-                    _logger.exception("Failed to create staging for branch %r", branch.name)
-                else:
-                    if commit:
-                        self.env.cr.commit()
+        # look up branches which can be staged on and have no active staging
+        for branch in self.env['runbot_merge.branch'].search([
+            ('active_staging_id', '=', False),
+            ('active', '=', True),
+            ('staging_enabled', '=', True),
+        ]):
+            try:
+                with self.env.cr.savepoint():
+                    branch.try_staging()
+            except Exception:
+                _logger.exception("Failed to create staging for branch %r", branch.name)
+            else:
+                if commit:
+                    self.env.cr.commit()
 
     def _find_commands(self, comment):
         return re.findall(
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 8be1070b..47ea4001 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -67,7 +67,7 @@ class Repository(models.Model):
 
     sequence = fields.Integer(default=50, group_operator=None)
     name = fields.Char(required=True)
-    project_id = fields.Many2one('runbot_merge.project', required=True)
+    project_id = fields.Many2one('runbot_merge.project', required=True, index=True)
     status_ids = fields.One2many('runbot_merge.repository.status', 'repo_id', string="Required Statuses")
 
     group_id = fields.Many2one('res.groups', default=lambda self: self.env.ref('base.group_user'))
@@ -235,10 +235,10 @@ class Branch(models.Model):
     _order = 'sequence, name'
 
     name = fields.Char(required=True)
-    project_id = fields.Many2one('runbot_merge.project', required=True)
+    project_id = fields.Many2one('runbot_merge.project', required=True, index=True)
 
     active_staging_id = fields.Many2one(
-        'runbot_merge.stagings', compute='_compute_active_staging', store=True,
+        'runbot_merge.stagings', compute='_compute_active_staging', store=True, index=True,
         help="Currently running staging for the branch."
     )
     staging_ids = fields.One2many('runbot_merge.stagings', 'target')
@@ -252,6 +252,8 @@ class Branch(models.Model):
     active = fields.Boolean(default=True)
     sequence = fields.Integer(group_operator=None)
 
+    staging_enabled = fields.Boolean(default=True)
+
     def _auto_init(self):
         res = super(Branch, self)._auto_init()
         tools.create_unique_index(
diff --git a/runbot_merge/tests/test_staging.py b/runbot_merge/tests/test_staging.py
new file mode 100644
index 00000000..c594cc76
--- /dev/null
+++ b/runbot_merge/tests/test_staging.py
@@ -0,0 +1,41 @@
+import pytest
+
+from utils import Commit, to_pr
+
+
+@pytest.fixture
+def repo(env, project, make_repo, users, setreviewers):
+    r = make_repo('repo')
+    project.write({'repo_ids': [(0, 0, {
+        'name': r.name,
+        'group_id': False,
+        'required_statuses': 'ci'
+    })]})
+    setreviewers(*project.repo_ids)
+    return r
+
+def test_staging_disabled_branch(env, project, repo, config):
+    """Check that it's possible to disable staging on a specific branch
+    """
+    project.branch_ids = [(0, 0, {
+        'name': 'other',
+        'staging_enabled': False,
+    })]
+    with repo:
+        [master_commit] = repo.make_commits(None, Commit("master", tree={'a': '1'}), ref="heads/master")
+        [c1] = repo.make_commits(master_commit, Commit("thing", tree={'a': '2'}), ref='heads/master-thing')
+        master_pr = repo.make_pr(title="whatever", target="master", head="master-thing")
+        master_pr.post_comment("hansen r+", config['role_reviewer']['token'])
+        repo.post_status(c1, 'success', 'ci')
+
+        [other_commit] = repo.make_commits(None, Commit("other", tree={'b': '1'}), ref='heads/other')
+        [c2] = repo.make_commits(other_commit, Commit("thing", tree={'b': '2'}), ref='heads/other-thing')
+        other_pr = repo.make_pr(title="whatever", target="other", head="other-thing")
+        other_pr.post_comment("hansen r+", config['role_reviewer']['token'])
+        repo.post_status(c2, 'success', 'ci')
+    env.run_crons()
+
+    assert to_pr(env, master_pr).staging_id, \
+        "master is allowed to stage, should be staged"
+    assert not to_pr(env, other_pr).staging_id, \
+        "other is *not* allowed to stage, should not be staged"
diff --git a/runbot_merge/views/res_partner.xml b/runbot_merge/views/res_partner.xml
index fde5f7e1..507e5a32 100644
--- a/runbot_merge/views/res_partner.xml
+++ b/runbot_merge/views/res_partner.xml
@@ -25,6 +25,7 @@
                     <field name="display_name" string="Name"/>
                     <field name="github_login"/>
                     <field name="review_rights" widget="many2many_tags"/>
+                    <field name="user_id" invisible="1"/>
                 </tree>
             </xpath>
         </field>
diff --git a/runbot_merge/views/runbot_merge_project.xml b/runbot_merge/views/runbot_merge_project.xml
index 2f670ed3..544f9cf9 100644
--- a/runbot_merge/views/runbot_merge_project.xml
+++ b/runbot_merge/views/runbot_merge_project.xml
@@ -56,7 +56,8 @@
                         <tree editable="bottom" decoration-muted="not active">
                             <field name="sequence" widget="handle" />
                             <field name="name"/>
-                            <field name="active"/>
+                            <field name="active" widget="boolean_toggle"/>
+                            <field name="staging_enabled" widget="boolean_toggle"/>
                         </tree>
                     </field>
                 </sheet>

From 048ae0c5ff9774f269d03b783a48b6449f42b097 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 8 Jun 2023 08:26:10 +0200
Subject: [PATCH 006/192] [FIX] forwardport: flag statuses as `recursive`

I'd been convinced this was an ORM error because the field is not
recursive... in runbot_merge, in forwardbot it is and thus does indeed
need to be flagged to avoid the warning.
---
 forwardport/models/project.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index fcf7c645..3112941b 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -213,6 +213,8 @@ class Repository(models.Model):
 class PullRequests(models.Model):
     _inherit = 'runbot_merge.pull_requests'
 
+    statuses = fields.Text(recursive=True)
+
     limit_id = fields.Many2one('runbot_merge.branch', help="Up to which branch should this PR be forward-ported")
 
     parent_id = fields.Many2one(

From e14616b2fb9a7458f4739c8c4212995c349e4b03 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 8 Jun 2023 10:03:26 +0200
Subject: [PATCH 007/192] [IMP] runbot_merge: add support for draft check

1cea247e6ce6b8add5bca06634655e43a53024dd missed the update of the
`draft` flag, add support for it.

Fixes #753
---
 runbot_merge/models/pull_requests.py | 9 ++++++++-
 runbot_merge/tests/test_basic.py     | 7 ++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 47ea4001..76ebb427 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -143,10 +143,17 @@ All substitutions are tentatively applied sequentially to the input.
                 },
                 'sender': {'login': self.project_id.github_prefix},
             })
+            edit2 = ''
+            if pr_id.draft != pr['draft']:
+                edit2 = controllers.handle_pr(self.env, {
+                    'action': 'converted_to_draft' if pr['draft'] else 'ready_for_review',
+                    'pull_request': pr,
+                    'sender': {'login': self.project_id.github_prefix}
+                }) + '. '
             feedback({
                 'repository': pr_id.repository.id,
                 'pull_request': number,
-                'message': f"{edit}. {sync}.",
+                'message': f"{edit}. {edit2}{sync}.",
             })
             return
 
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index ea5dd098..c15925b1 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -2520,6 +2520,7 @@ Please check and re-approve.
             'state': 'ready',
             'message': "Something else",
             'target': other.id,
+            'draft': True,
         })
         with repo:
             pr.post_comment('hansen check')
@@ -2528,7 +2529,11 @@ Please check and re-approve.
         assert pr_id.head == c2
         assert pr_id.message == 'title\n\nbody' # the commit's message was used for the PR
         assert pr_id.target.name == 'master'
-        assert pr.comments[-1] == (users['user'], f"Updated target, squash, message. Updated to {c2}.")
+        assert not pr_id.draft
+        assert pr.comments[-1] == (
+            users['user'],
+            f"Updated target, squash, message. Updated {pr_id.display_name} to ready. Updated to {c2}."
+        )
 
     def test_update_closed(self, env, repo):
         with repo:

From 270dfdd495e5f58ee78f245aa76f1b49a05bda4d Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 12 Jun 2023 14:41:42 +0200
Subject: [PATCH 008/192] [REF] *: move most feedback messages to
 pseudo-templates

Current system makes it hard to iterate feedback messages and make
them clearer, this should improve things a touch.

Use a bespoke model to avoid concerns with qweb rendering
complexity (we just want GFM output and should not need logic).

Also update fwbot test setup to always configure an fwbot name, in
order to avoid ping messages closing the PRs they're talking
about, that took a while to debug, and given the old message I assume
I'd already hit it and just been too lazy to fix. This requires
updating a bunch of tests as fwbot ping are sent *to*
`fp_github_name`, but sent *from* the reference user (because that's
the key we set).

Note: noupdate on CSV files doesn't seem to work anymore, which isn't
great. But instead set tracking on the template's templates, it's not
quite as good but should be sufficient.

Fixes #769
---
 forwardport/models/forwardport.py             |  61 ++---
 forwardport/models/project.py                 | 213 +++++++--------
 forwardport/tests/test_conflicts.py           |   6 +-
 forwardport/tests/test_limit.py               |  13 +-
 forwardport/tests/test_simple.py              |  20 +-
 forwardport/tests/test_updates.py             |   5 +-
 forwardport/tests/test_weird.py               |  10 +-
 mergebot_test_utils/utils.py                  |   2 +
 runbot_merge/__manifest__.py                  |   1 +
 runbot_merge/controllers/__init__.py          |  14 +-
 ..._merge.pull_requests.feedback.template.csv | 171 ++++++++++++
 runbot_merge/models/pull_requests.py          | 246 +++++++++---------
 runbot_merge/security/ir.model.access.csv     |   1 +
 runbot_merge/tests/test_basic.py              |  11 +-
 runbot_merge/tests/test_disabled_branch.py    |   2 +-
 runbot_merge/views/configuration.xml          |  37 ++-
 16 files changed, 512 insertions(+), 301 deletions(-)
 create mode 100644 runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv

diff --git a/forwardport/models/forwardport.py b/forwardport/models/forwardport.py
index f94b6db9..78c23a46 100644
--- a/forwardport/models/forwardport.py
+++ b/forwardport/models/forwardport.py
@@ -98,14 +98,6 @@ class ForwardPortTasks(models.Model, Queue):
         batch.active = False
 
 
-CONFLICT_TEMPLATE = "{ping}WARNING: the latest change ({previous.head}) triggered " \
-                    "a conflict when updating the next forward-port " \
-                    "({next.display_name}), and has been ignored.\n\n" \
-                    "You will need to update this pull request differently, " \
-                    "or fix the issue by hand on {next.display_name}."
-CHILD_CONFLICT = "{ping}WARNING: the update of {previous.display_name} to " \
-                 "{previous.head} has caused a conflict in this pull request, " \
-                 "data may have been lost."
 class UpdateQueue(models.Model, Queue):
     _name = 'forwardport.updates'
     _description = 'if a forward-port PR gets updated & has followups (cherrypick succeeded) the followups need to be updated as well'
@@ -116,7 +108,6 @@ class UpdateQueue(models.Model, Queue):
     new_root = fields.Many2one('runbot_merge.pull_requests')
 
     def _process_item(self):
-        Feedback = self.env['runbot_merge.pull_requests.feedback']
         previous = self.new_root
         with ExitStack() as s:
             for child in self.new_root._iter_descendants():
@@ -134,41 +125,35 @@ class UpdateQueue(models.Model, Queue):
                     self.new_root.display_name
                 )
                 if child.state in ('closed', 'merged'):
-                    Feedback.create({
-                        'repository': child.repository.id,
-                        'pull_request': child.number,
-                        'message': "%sancestor PR %s has been updated but this PR"
-                                   " is %s and can't be updated to match."
-                                   "\n\n"
-                                   "You may want or need to manually update any"
-                                   " followup PR." % (
-                            child.ping(),
-                            self.new_root.display_name,
-                            child.state,
-                        )
-                    })
+                    self.env.ref('runbot_merge.forwardport.updates.closed')._send(
+                        repository=child.repository,
+                        pull_request=child.number,
+                        token_field='fp_github_token',
+                        format_args={'pr': child, 'parent': self.new_root},
+                    )
                     return
 
                 conflicts, working_copy = previous._create_fp_branch(
                     child.target, child.refname, s)
                 if conflicts:
                     _, out, err, _ = conflicts
-                    Feedback.create({
-                        'repository': previous.repository.id,
-                        'pull_request': previous.number,
-                        'message': CONFLICT_TEMPLATE.format(
-                            ping=previous.ping(),
-                            previous=previous,
-                            next=child
-                        )
-                    })
-                    Feedback.create({
-                        'repository': child.repository.id,
-                        'pull_request': child.number,
-                        'message': CHILD_CONFLICT.format(ping=child.ping(), previous=previous, next=child)\
-                            + (f'\n\nstdout:\n```\n{out.strip()}\n```' if out.strip() else '')
-                            + (f'\n\nstderr:\n```\n{err.strip()}\n```' if err.strip() else '')
-                    })
+                    self.env.ref('runbot_merge.forwardport.updates.conflict.parent')._send(
+                        repository=previous.repository,
+                        pull_request=previous.number,
+                        token_field='fp_github_token',
+                        format_args={'pr': previous, 'next': child},
+                    )
+                    self.env.ref('runbot_merge.forwardport.updates.conflict.child')._send(
+                        repository=child.repository,
+                        pull_request=child.number,
+                        token_field='fp_github_token',
+                        format_args={
+                            'previous': previous,
+                            'pr': child,
+                            'stdout': (f'\n\nstdout:\n```\n{out.strip()}\n```' if out.strip() else ''),
+                            'stderr': (f'\n\nstderr:\n```\n{err.strip()}\n```' if err.strip() else ''),
+                        },
+                    )
 
                 new_head = working_copy.stdout().rev_parse(child.refname).stdout.decode().strip()
                 commits_count = int(working_copy.stdout().rev_list(
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 3112941b..720000e7 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -67,8 +67,9 @@ class Project(models.Model):
     def _compute_git_identity(self):
         s = requests.Session()
         for project in self:
-            if not project.fp_github_token:
+            if not project.fp_github_token or (self.fp_github_name and self.fp_github_email):
                 continue
+
             r0 = s.get('https://api.github.com/user', headers={
                 'Authorization': 'token %s' % project.fp_github_token
             })
@@ -246,6 +247,25 @@ class PullRequests(models.Model):
         for pr in self:
             pr.refname = pr.label.split(':', 1)[-1]
 
+    ping = fields.Char(recursive=True)
+
+    @api.depends('source_id.author.github_login', 'source_id.reviewed_by.github_login')
+    def _compute_ping(self):
+        """For forward-port PRs (PRs with a source) the author is the PR bot, so
+        we want to ignore that and use the author & reviewer of the original PR
+        """
+        source = self.source_id
+        if not source:
+            return super()._compute_ping()
+
+        for pr in self:
+            s = ' '.join(
+                f'@{p.github_login}'
+                for p in source.author | source.reviewed_by | self.reviewed_by
+            )
+            pr.ping = s and (s + ' ')
+
+
     @api.model_create_single
     def create(self, vals):
         # PR opened event always creates a new PR, override so we can precreate PRs
@@ -297,32 +317,24 @@ class PullRequests(models.Model):
         if self.env.context.get('forwardport_detach_warn', True):
             for p in with_parents:
                 if not p.parent_id:
-                    self.env['runbot_merge.pull_requests.feedback'].create({
-                        'repository': p.repository.id,
-                        'pull_request': p.number,
-                        'message': "%sthis PR was modified / updated and has become a normal PR. "
-                                   "It should be merged the normal way (via @%s)" % (
-                            p.source_id.ping(),
-                            p.repository.project_id.github_prefix,
-                        ),
-                        'token_field': 'fp_github_token',
-                    })
+                    self.env.ref('runbot_merge.forwardport.update.detached')._send(
+                        repository=p.repository,
+                        pull_request=p.number,
+                        token_field='fp_github_token',
+                        format_args={'pr': p},
+                    )
         for p in closed_fp.filtered(lambda p: p.state != 'closed'):
-            self.env['runbot_merge.pull_requests.feedback'].create({
-                'repository': p.repository.id,
-                'pull_request': p.number,
-                'message': "%sthis PR was closed then reopened. "
-                           "It should be merged the normal way (via @%s)" % (
-                    p.source_id.ping(),
-                    p.repository.project_id.github_prefix,
-                ),
-                'token_field': 'fp_github_token',
-            })
+            self.env.ref('runbot_merge.forwardport.reopen.detached')._send(
+                repository=p.repository,
+                pull_request=p.number,
+                token_field='fp_github_token',
+                format_args={'pr': p},
+            )
         if vals.get('state') == 'merged':
-            for p in self:
-                self.env['forwardport.branch_remover'].create({
-                    'pr_id': p.id,
-                })
+            self.env['forwardport.branch_remover'].create([
+                {'pr_id': p.id}
+                for p in self
+            ])
         # if we change the policy to skip CI, schedule followups on existing FPs
         if vals.get('fw_policy') == 'skipci' and self.state == 'merged':
             self.env['runbot_merge.pull_requests'].search([
@@ -455,15 +467,12 @@ class PullRequests(models.Model):
         if not (self.state == 'opened' and self.parent_id):
             return
 
-        self.env['runbot_merge.pull_requests.feedback'].create({
-            'repository': self.repository.id,
-            'pull_request': self.number,
-            'token_field': 'fp_github_token',
-            'message': '%s%s failed on this forward-port PR' % (
-                self.source_id.ping(),
-                ci,
-            )
-        })
+        self.env.ref('runbot_merge.forwardport.ci.failed')._send(
+            repository=self.repository,
+            pull_request=self.number,
+            token_field='fp_github_token',
+            format_args={'pr': self, 'ci': ci},
+        )
 
     def _validate(self, statuses):
         failed = super()._validate(statuses)
@@ -636,16 +645,12 @@ class PullRequests(models.Model):
                 linked, other = different_pr, different_target
                 if t != target:
                     linked, other = ref, target
-                self.env['runbot_merge.pull_requests.feedback'].create({
-                    'repository': pr.repository.id,
-                    'pull_request': pr.number,
-                    'token_field': 'fp_github_token',
-                    'message': "%sthis pull request can not be forward ported: "
-                               "next branch is %r but linked pull request %s "
-                               "has a next branch %r." % (
-                        pr.ping(), t.name, linked.display_name, other.name
-                    )
-                })
+                self.env.ref('runbot_merge.forwardport.failure.discrepancy')._send(
+                    repository=pr.repository,
+                    pull_request=pr.number,
+                    token_field='fp_github_token',
+                    format_args={'pr': pr, 'linked': linked, 'next': t.name, 'other': other.name},
+                )
             _logger.warning(
                 "Cancelling forward-port of %s: found different next branches (%s)",
                 self, all_targets
@@ -743,23 +748,18 @@ class PullRequests(models.Model):
                 'delegates': [(6, False, (source.delegates | pr.delegates).ids)]
             })
             if has_conflicts and pr.parent_id and pr.state not in ('merged', 'closed'):
-                message = source.ping() + """\
-the next pull request (%s) is in conflict. You can merge the chain up to here by saying
-> @%s r+
-%s""" % (new_pr.display_name, pr.repository.project_id.fp_github_name, footer)
-                self.env['runbot_merge.pull_requests.feedback'].create({
-                    'repository': pr.repository.id,
-                    'pull_request': pr.number,
-                    'message': message,
-                    'token_field': 'fp_github_token',
-                })
+                self.env.ref('runbot_merge.forwardport.failure.conflict')._send(
+                    repository=pr.repository,
+                    pull_request=pr.number,
+                    token_field='fp_github_token',
+                    format_args={'source': source, 'pr': pr, 'new': new_pr, 'footer': footer},
+                )
             # not great but we probably want to avoid the risk of the webhook
             # creating the PR from under us. There's still a "hole" between
             # the POST being executed on gh and the commit but...
             self.env.cr.commit()
 
         for pr, new_pr in zip(self, new_batch):
-            source = pr.source_id or pr
             (h, out, err, hh) = conflicts.get(pr) or (None, None, None, None)
 
             if h:
@@ -775,47 +775,47 @@ the next pull request (%s) is in conflict. You can merge the chain up to here by
                         '* %s%s\n' % (sha, ' <- on this commit' if sha == h else '')
                         for sha in hh
                     )
-                message = f"""{source.ping()}cherrypicking of pull request {source.display_name} failed.
-{lines}{sout}{serr}
-Either perform the forward-port manually (and push to this branch, proceeding as usual) or close this PR (maybe?).
-
-In the former case, you may want to edit this PR message as well.
-"""
+                template = 'runbot_merge.forwardport.failure'
+                format_args = {
+                    'pr': new_pr,
+                    'commits': lines,
+                    'stdout': sout,
+                    'stderr': serr,
+                    'footer': footer,
+                }
             elif has_conflicts:
-                message = """%s\
-while this was properly forward-ported, at least one co-dependent PR (%s) did \
-not succeed. You will need to fix it before this can be merged.
-
-Both this PR and the others will need to be approved via `@%s r+` as they are \
-all considered "in conflict".
-%s""" % (
-                    source.ping(),
-                    ', '.join(p.display_name for p in (new_batch - new_pr)),
-                    proj.github_prefix,
-                    footer
-                )
+                template = 'runbot_merge.forwardport.linked'
+                format_args = {
+                    'pr': new_pr,
+                    'siblings': ', '.join(p.display_name for p in (new_batch - new_pr)),
+                    'footer': footer,
+                }
             elif base._find_next_target(new_pr) is None:
                 ancestors = "".join(
                     "* %s\n" % p.display_name
                     for p in pr._iter_ancestors()
                     if p.parent_id
                 )
-                message = source.ping() + """\
-this PR targets %s and is the last of the forward-port chain%s
-%s
-To merge the full chain, say
-> @%s r+
-%s""" % (target.name, ' containing:' if ancestors else '.', ancestors, pr.repository.project_id.fp_github_name, footer)
+                template = 'runbot_merge.forwardport.final'
+                format_args = {
+                    'pr': new_pr,
+                    'containing': ' containing:' if ancestors else '.',
+                    'ancestors': ancestors,
+                    'footer': footer,
+                }
             else:
-                message = """\
-This PR targets %s and is part of the forward-port chain. Further PRs will be created up to %s.
-%s""" % (target.name, base.limit_id.name, footer)
-            self.env['runbot_merge.pull_requests.feedback'].create({
-                'repository': new_pr.repository.id,
-                'pull_request': new_pr.number,
-                'message': message,
-                'token_field': 'fp_github_token',
-            })
+                template = 'runbot_merge.forwardport.intermediate'
+                format_args = {
+                    'pr': new_pr,
+                    'footer': footer,
+                }
+            self.env.ref(template)._send(
+                repository=new_pr.repository,
+                pull_request=new_pr.number,
+                token_field='fp_github_token',
+                format_args=format_args,
+            )
+
             labels = ['forwardport']
             if has_conflicts:
                 labels.append('conflict')
@@ -1150,31 +1150,18 @@ stderr:
             if source.merge_date > (cutoff_dt - backoff):
                 continue
             source.reminder_backoff_factor += 1
-            self.env['runbot_merge.pull_requests.feedback'].create({
-                'repository': source.repository.id,
-                'pull_request': source.number,
-                'message': "%sthis pull request has forward-port PRs awaiting action (not merged or closed):\n%s" % (
-                    source.ping(),
-                    '\n- '.join(pr.display_name for pr in sorted(prs, key=lambda p: p.number))
-                ),
-                'token_field': 'fp_github_token',
-            })
-
-    def ping(self, author=True, reviewer=True):
-        source = self.source_id
-        if not source:
-            return super().ping(author=author, reviewer=reviewer)
-
-        # use a dict literal to maintain ordering (3.6+)
-        pingline = ' '.join(
-            f'@{p.github_login}'
-            for p in filter(None, {
-                author and source.author: None,
-                reviewer and source.reviewed_by: None,
-                reviewer and self.reviewed_by: None,
-            })
-        )
-        return pingline and (pingline + ' ')
+            self.env.ref('runbot_merge.forwardport.reminder')._send(
+                repository=source.repository,
+                pull_request=source.number,
+                token_field='fp_github_token',
+                format_args={
+                    'pr': source,
+                    'outstanding': ''.join(
+                        f'\n- {pr.display_name}'
+                        for pr in sorted(prs, key=lambda p: p.number)
+                    ),
+                }
+            )
 
 class Stagings(models.Model):
     _inherit = 'runbot_merge.stagings'
diff --git a/forwardport/tests/test_conflicts.py b/forwardport/tests/test_conflicts.py
index 3cda8430..d5bb3486 100644
--- a/forwardport/tests/test_conflicts.py
+++ b/forwardport/tests/test_conflicts.py
@@ -316,11 +316,11 @@ def test_multiple_commits_different_authorship(env, config, make_repo, users, ro
     c = prod.commit(pr2_id.head)
     assert len(c.parents) == 1
     get = itemgetter('name', 'email')
-    rm = rolemap['user']
-    assert get(c.author) == (rm['login'], ''), \
+    bot = pr_id.repository.project_id.fp_github_name
+    assert get(c.author) == (bot, ''), \
         "In a multi-author PR, the squashed conflict commit should have the " \
         "author set to the bot but an empty email"
-    assert get(c.committer) == (rm['login'], '')
+    assert get(c.committer) == (bot, '')
 
     assert re.match(r'''<<<\x3c<<< HEAD
 b
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index fca04111..db13f535 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -174,7 +174,7 @@ def test_limit_after_merge(env, config, make_repo, users):
         (users['reviewer'], "hansen r+"),
         seen(env, pr1, users),
         (users['reviewer'], bot_name + ' up to b'),
-        (bot_name, "@%s forward-port limit can only be set before the PR is merged." % users['reviewer']),
+        (users['user'], "@%s forward-port limit can only be set before the PR is merged." % users['reviewer']),
     ]
     assert pr2.comments == [
         seen(env, pr2, users),
@@ -184,7 +184,7 @@ This PR targets b and is part of the forward-port chain. Further PRs will be cre
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 """),
         (users['reviewer'], bot_name + ' up to b'),
-        (bot_name, "@%s forward-port limit can only be set on an origin PR"
+        (users['user'], "@%s forward-port limit can only be set on an origin PR"
                    " (%s here) before it's merged and forward-ported." % (
             users['reviewer'],
             p1.display_name,
@@ -208,13 +208,14 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
     env.run_crons()
 
     assert pr2.comments[4:] == [
-        (bot_name, "@%s @%s this PR was modified / updated and has become a normal PR. "
+        (users['user'], "@%s @%s this PR was modified / updated and has become a normal PR. "
                    "It should be merged the normal way (via @%s)" % (
             users['user'], users['reviewer'],
             p2.repository.project_id.github_prefix
         )),
         (users['reviewer'], bot_name + ' up to b'),
-        (bot_name, f"@{users['reviewer']} forward-port limit can only be set on an origin PR "
-                   f"({p1.display_name} here) before it's merged and forward-ported."
-         ),
+        (users['user'], f"@{users['reviewer']} forward-port limit can only be "
+                        f"set on an origin PR ({p1.display_name} here) before "
+                        f"it's merged and forward-ported."
+        ),
     ]
diff --git a/forwardport/tests/test_simple.py b/forwardport/tests/test_simple.py
index ef31c495..5f5e21b8 100644
--- a/forwardport/tests/test_simple.py
+++ b/forwardport/tests/test_simple.py
@@ -109,7 +109,7 @@ def test_straightforward_flow(env, config, make_repo, users):
     assert c.author['name'] == other_user['user'], "author should still be original's probably"
     assert c.committer['name'] == other_user['user'], "committer should also still be the original's, really"
 
-    assert pr1.ping() == "@%s @%s " % (
+    assert pr1.ping == "@%s @%s " % (
         config['role_other']['user'],
         config['role_reviewer']['user'],
     ), "ping of forward-port PR should include author and reviewer of source"
@@ -132,7 +132,7 @@ def test_straightforward_flow(env, config, make_repo, users):
         (users['reviewer'], 'hansen r+ rebase-ff'),
         seen(env, pr, users),
         (users['user'], 'Merge method set to rebase and fast-forward.'),
-        (users['user'], '@%s @%s this pull request has forward-port PRs awaiting action (not merged or closed):\n%s' % (
+        (users['user'], '@%s @%s this pull request has forward-port PRs awaiting action (not merged or closed):\n- %s' % (
             users['other'], users['reviewer'],
             '\n- '.join((pr1 | pr2).mapped('display_name'))
         )),
@@ -160,7 +160,7 @@ def test_straightforward_flow(env, config, make_repo, users):
 @%s @%s this PR targets c and is the last of the forward-port chain containing:
 * %s
 
-To merge the full chain, say
+To merge the full chain, use
 > @%s r+
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
@@ -315,7 +315,11 @@ def test_empty(env, config, make_repo, users):
     assert env['runbot_merge.pull_requests'].search([], order='number') == prs
     # change FP token to see if the feedback comes from the proper user
     project = env['runbot_merge.project'].search([])
-    project.fp_github_token = config['role_other']['token']
+    project.write({
+        'fp_github_name': False,
+        'fp_github_email': False,
+        'fp_github_token': config['role_other']['token'],
+    })
     assert project.fp_github_name == users['other']
 
     # check reminder
@@ -324,7 +328,7 @@ def test_empty(env, config, make_repo, users):
 
     awaiting = (
         users['other'],
-        '@%s @%s this pull request has forward-port PRs awaiting action (not merged or closed):\n%s' % (
+        '@%s @%s this pull request has forward-port PRs awaiting action (not merged or closed):\n- %s' % (
             users['user'], users['reviewer'],
             fail_id.display_name
         )
@@ -582,11 +586,11 @@ def test_delegate_fw(env, config, make_repo, users):
         seen(env, pr2, users),
         (users['user'], '''@{self_reviewer} @{reviewer} this PR targets c and is the last of the forward-port chain.
 
-To merge the full chain, say
-> @{user} r+
+To merge the full chain, use
+> @{bot} r+
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
-'''.format_map(users)),
+'''.format(bot=pr1_id.repository.project_id.fp_github_name, **users)),
         (users['other'], 'hansen r+')
     ]
 
diff --git a/forwardport/tests/test_updates.py b/forwardport/tests/test_updates.py
index 384df08f..1635fc5e 100644
--- a/forwardport/tests/test_updates.py
+++ b/forwardport/tests/test_updates.py
@@ -3,9 +3,6 @@ Test cases for updating PRs during after the forward-porting process after the
 initial merge has succeeded (and forward-porting has started)
 """
 import re
-import sys
-
-import pytest
 
 from utils import seen, re_matches, Commit, make_basic, to_pr
 
@@ -248,6 +245,8 @@ def test_duplicate_fw(env, make_repo, setreviewers, config, users):
         'github_token': config['github']['token'],
         'github_prefix': 'hansen',
         'fp_github_token': config['github']['token'],
+        'fp_github_name': 'herbert',
+        'fp_github_email': 'hb@example.com',
         'branch_ids': [
             (0, 0, {'name': 'master', 'sequence': 0}),
             (0, 0, {'name': 'v3', 'sequence': 1}),
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index 4ef56feb..32bcf0db 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -25,6 +25,8 @@ def make_basic(env, config, make_repo, *, fp_token, fp_remote):
             'github_token': config['github']['token'],
             'github_prefix': 'hansen',
             'fp_github_token': fp_token and config['github']['token'],
+            'fp_github_name': 'herbert',
+            'fp_github_email': 'hb@example.com',
             'branch_ids': [
                 (0, 0, {'name': 'a', 'sequence': 2}),
                 (0, 0, {'name': 'b', 'sequence': 1}),
@@ -262,6 +264,8 @@ class TestNotAllBranches:
             'github_token': config['github']['token'],
             'github_prefix': 'hansen',
             'fp_github_token': config['github']['token'],
+            'fp_github_name': 'herbert',
+            'fp_github_email': 'hb@example.com',
             'branch_ids': [
                 (0, 0, {'name': 'a', 'sequence': 2}),
                 (0, 0, {'name': 'b', 'sequence': 1}),
@@ -401,7 +405,7 @@ class TestNotAllBranches:
         assert pr_a.comments == [
             (users['reviewer'], 'hansen r+'),
             seen(env, pr_a, users),
-            (users['user'], "@%s @%s this pull request can not be forward ported:"
+            (users['user'], "@%s @%s this pull request can not be forward-ported:"
                             " next branch is 'b' but linked pull request %s "
                             "has a next branch 'c'." % (
                 users['user'], users['reviewer'], pr_b_id.display_name,
@@ -410,7 +414,7 @@ class TestNotAllBranches:
         assert pr_b.comments == [
             (users['reviewer'], 'hansen r+'),
             seen(env, pr_b, users),
-            (users['user'], "@%s @%s this pull request can not be forward ported:"
+            (users['user'], "@%s @%s this pull request can not be forward-ported:"
                             " next branch is 'c' but linked pull request %s "
                             "has a next branch 'b'." % (
                 users['user'], users['reviewer'], pr_a_id.display_name,
@@ -472,6 +476,7 @@ def test_new_intermediate_branch(env, config, make_repo):
     with prod:
         validate(prod, pr0_fp_id.head)
     env.run_crons()
+    assert pr0_fp_id.state == 'validated'
     original0 = PRs.search([('parent_id', '=', pr0_fp_id.id)])
     assert original0, "Could not find FP of PR0 to C"
     assert original0.target.name == 'c'
@@ -519,6 +524,7 @@ def test_new_intermediate_branch(env, config, make_repo):
     })
     env.run_crons()
 
+    assert pr0_fp_id.state == 'validated'
     # created an intermediate PR for 0 and x
     desc0 = PRs.search([('source_id', '=', pr0_id.id)])
     new0 = desc0 - pr0_fp_id - original0
diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index a23d6726..b87a2345 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -73,6 +73,8 @@ def make_basic(env, config, make_repo, *, reponame='proj', project_name='myproje
             'github_token': config['github']['token'],
             'github_prefix': 'hansen',
             'fp_github_token': config['github']['token'],
+            'fp_github_name': 'herbert',
+            'fp_github_email': 'hb@example.com',
             'branch_ids': [
                 (0, 0, {'name': 'a', 'sequence': 100}),
                 (0, 0, {'name': 'b', 'sequence': 80}),
diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py
index 82978d76..3640d249 100644
--- a/runbot_merge/__manifest__.py
+++ b/runbot_merge/__manifest__.py
@@ -7,6 +7,7 @@
         'security/ir.model.access.csv',
 
         'data/merge_cron.xml',
+        'data/runbot_merge.pull_requests.feedback.template.csv',
         'views/res_partner.xml',
         'views/runbot_merge_project.xml',
         'views/mergebot.xml',
diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index efb752ac..51d8c71d 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -143,11 +143,19 @@ def handle_pr(env, event):
 
     message = None
     if not branch:
-        message = f"This PR targets the un-managed branch {r}:{b}, it needs to be retargeted before it can be merged."
+        message = env.ref('runbot_merge.handle.branch.unmanaged')._format(
+            repository=r,
+            branch=b,
+            event=event,
+        )
         _logger.info("Ignoring event %s on PR %s#%d for un-managed branch %s",
                      event['action'], r, pr['number'], b)
     elif not branch.active:
-        message = f"This PR targets the disabled branch {r}:{b}, it needs to be retargeted before it can be merged."
+        message = env.ref('runbot_merge.handle.branch.inactive')._format(
+            repository=r,
+            branch=b,
+            event=event,
+        )
     if message and event['action'] not in ('synchronize', 'closed'):
         feedback(message=message)
 
@@ -240,7 +248,7 @@ def handle_pr(env, event):
         if pr_obj.state == 'merged':
             feedback(
                 close=True,
-                message="@%s ya silly goose you can't reopen a merged PR." % event['sender']['login']
+                message=env.ref('runbot_merge.handle.pr.merged')._format(event=event),
             )
 
         if pr_obj.state == 'closed':
diff --git a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
new file mode 100644
index 00000000..70ed7397
--- /dev/null
+++ b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
@@ -0,0 +1,171 @@
+id,template,help
+runbot_merge.handle.branch.unmanaged,"This PR targets the un-managed branch {repository}:{branch}, it needs to be retargeted before it can be merged.","Notifies of event on PR whose branch is not managed  by the mergebot.
+
+repository: repository name
+branch: branch (ref) name
+event: complete pr event"
+runbot_merge.handle.branch.inactive,"This PR targets the disabled branch {repository}:{branch}, it needs to be retargeted before it can be merged.","Notifies of event on PR whose branch is deactivated.
+
+repository: repository name
+branch: branch (ref) name
+event: complete pr event"
+runbot_merge.handle.pr.merged,@{event[sender][login]} ya silly goose you can't reopen a merged PR.,"Notifies that a user tried to reopen a merged PR.
+
+Event: complete PR event"
+runbot_merge.pr.load.unmanaged,"Branch `{pr[base][ref]}` is not within my remit, imma just ignore it.","Notifies that a user tried to load a PR targeting a non-handled branch.
+
+pr: pull request (github object)
+Repository: repository object (???)"
+runbot_merge.pr.load.fetched,"{pr.ping}I didn't know about this PR and had to retrieve its information, you may have to re-approve it as I didn't see previous commands.","Notifies that we did retrieve an unknown PR (either by request or as side effect of an interaction).
+
+Pr: pr object we just created"
+runbot_merge.pr.branch.disabled,"{pr.ping}the target branch {pr.target.name!r} has been disabled, you may want to close this PR.","Notifies that the target branch for this PR was deactivated.
+
+pr: pull request in question"
+runbot_merge.pr.merge.failed,{pr.ping}unable to stage: {reason},"Notifies that the PR could not be merged into the staging branch.
+
+pr: pr object we tried to merge
+reason: error message
+exc: exception object"
+runbot_merge.pr.fetch.unmanaged,I'm sorry. Branch `{branch}` is not within my remit.,"Responds to a request to fetch a PR to an unmanaged branch.
+
+repository: pr repository
+branch: target branch
+number: pr number"
+runbot_merge.command.access.no,"I'm sorry, @{user}. I'm afraid I can't do that.","Responds to command by a user who has no rights at all.
+
+user: github login of comment sender
+pr: pr object to which the command was sent"
+runbot_merge.command.approve.failure,@{user} you may want to rebuild or fix this PR as it has failed CI.,"Responds to r+ of PR with failed CI.
+
+user: github login of comment sender
+pr: pr object to which the command was sent"
+runbot_merge.command.unapprove.p0,"PR priority reset to 1, as pull requests with priority 0 ignore review state.","Responds to r- of pr in p=0.
+
+user: github login of comment sender
+pr: pr object to which the command was sent"
+runbot_merge.command.method,Merge method set to {new_method}.,"Responds to the setting of the merge method.
+
+new_method: ...
+pr: pr object to which the command was sent
+user: github login of the comment sender"
+runbot_merge.failure.approved,{pr.ping}{status!r} failed on this reviewed PR.,"Notification of failed status on a reviewed PR.
+
+pr: pull request in question
+status: failed status"
+runbot_merge.pr.created,[Pull request status dashboard]({pr.url}).,"Initial comment on PR creation.
+
+pr: created pr"
+runbot_merge.pr.linked.not_ready,{pr.ping}linked pull request(s) {siblings} not ready. Linked PRs are not staged until all of them are ready.,"Comment when a PR is ready (approved & validated) but it is linked to other PRs which are not.
+
+pr: pr we're looking at
+siblings: its siblings, as a single comma-separated list of PR links"
+runbot_merge.pr.merge_method,"{pr.ping}because this PR has multiple commits, I need to know how to merge it:
+
+{methods}","Comment when a PR is ready but doesn't have a merge method set
+
+pr: the pr we can't stage
+methods: a markdown-formatted list of valid merge methods"
+runbot_merge.pr.staging.mismatch,"{pr.ping}we apparently missed updates to this PR and tried to stage it in a state which might not have been approved.
+
+The properties {mismatch} were not correctly synchronized and have been updated.
+
+<details><summary>differences</summary>
+
+```diff
+{diff}```
+</details>
+
+Note that we are unable to check the properties {unchecked}.
+
+Please check and re-approve.
+","Comment when staging was attempted but a sanity check revealed the github state and the mergebot state differ.
+
+pr: the pr we tried to stage
+mismatch: comma separated list of mismatched property names
+diff: patch-style view of the differing properties
+unchecked: comma-separated list of properties which can't be checked"
+runbot_merge.pr.staging.fail,{pr.ping}staging failed: {message},"Comment when a PR caused a staging to fail (normally only sent if  the staging has a single batch, may be sent on multiple PRs depending whether the heuristic to guess the problematic PR of a batch succeeded)
+
+pr: the pr
+message: staging failure information (error message, build link, etc...)"
+runbot_merge.forwardport.updates.closed,"{pr.ping}ancestor PR {parent.display_name} has been updated but this PR is {pr.state} and can't be updated to match.
+
+You may want or need to manually update any followup PR.","Comment when a PR is updated and on of its followups is already merged or closed. Sent to the followup.
+
+pr: the closed or merged PR
+parent: the modified ancestor PR"
+runbot_merge.forwardport.updates.conflict.parent,"{pr.ping}WARNING: the latest change ({pr.head}) triggered a conflict when updating the next forward-port ({next.display_name}), and has been ignored.
+
+You will need to update this pull request differently, or fix the issue by hand on {next.display_name}.","Comment when a PR update triggers a conflict in a child.
+
+pr: updated parent PR
+next: child PR in conflict"
+runbot_merge.forwardport.updates.conflict.child,"{pr.ping}WARNING: the update of {previous.display_name} to {previous.head} has caused a conflict in this pull request, data may have been lost.{stdout}{stderr}","Comment when a PR update followup is in conflict.
+
+pr: PR where update followup conflict happened
+previous: parent PR which triggered the followup
+stdout: markdown-formatted stdout of git, if any
+stderr: markdown-formatted stderr of git, if any"
+runbot_merge.forwardport.update.detached,{pr.ping}this PR was modified / updated and has become a normal PR. It should be merged the normal way (via @{pr.repository.project_id.github_prefix}),"Comment when a forwardport PR gets updated, documents that the PR now needs to be merged the “normal” way.
+
+pr: the pr in question "
+runbot_merge.forwardport.reopen.detached,{pr.ping}this PR was closed then reopened. It should be merged the normal way (via @{pr.repository.project_id.github_prefix}),"Comment when a forwardport PR gets closed then reopened, documents that the PR is now in a detached state.
+
+pr: the pr in question"
+runbot_merge.forwardport.ci.failed,{pr.ping}{ci} failed on this forward-port PR,"Comment when CI fails on a forward-port PR (which thus won't port any further, for now).
+
+pr: the pr in question
+ci: the failed status"
+runbot_merge.forwardport.failure.discrepancy,{pr.ping}this pull request can not be forward-ported: next branch is {next!r} but linked pull request {linked.display_name} has a next branch {other!r}.,"Comment when we tried to forward port a PR batch, but the PRs have different next targets (unlikely to happen really).
+
+pr: the pr we tried to forward port
+linked: the linked PR with a different next target
+next: next target for the current pr
+other: next target for the other pr"
+runbot_merge.forwardport.failure.conflict,"{pr.ping}the next pull request ({new.display_name}) is in conflict. You can merge the chain up to here by saying
+> @{pr.repository.project_id.fp_github_name} r+
+{footer}","Comment when a forward port was created but is in conflict, warns of that & gives instructions for current PR.
+
+pr: the pr which was just forward ported
+new: the new forward-port
+footer: some footer text"
+runbot_merge.forwardport.reminder,{pr.ping}this pull request has forward-port PRs awaiting action (not merged or closed):{outstanding},"Comment when a forward port has outstanding (not merged or closed) descendants
+
+pr: the original (source) PR
+outstanding: markdown-formatted list of outstanding PRs"
+runbot_merge.forwardport.failure,"{pr.ping}cherrypicking of pull request {pr.source_id.display_name} failed.
+{commits}{stdout}{stderr}
+Either perform the forward-port manually (and push to this branch, proceeding as usual) or close this PR (maybe?).
+
+In the former case, you may want to edit this PR message as well.
+{footer}","Comment when a forward-port failed.
+
+pr: the new pr (in failure)
+commits: markdown-formatted list of source commits, indicating which failed
+stdout: git's stdout
+stderr: git's stderr
+footer: some footer text"
+runbot_merge.forwardport.linked,"{pr.ping}while this was properly forward-ported, at least one co-dependent PR ({siblings}) did not succeed. You will need to fix it before this can be merged.
+
+Both this PR and the others will need to be approved via `@{pr.repository.project_id.github_prefix} r+` as they are all considered “in conflict”.
+{footer} ","Comment when a forward port succeeded but at least one sibling failed.
+
+pr: the current pr (new)
+siblings: comma-separated list of sibling links
+footer: some footer text"
+runbot_merge.forwardport.final,"{pr.ping}this PR targets {pr.target.name} and is the last of the forward-port chain{containing}
+{ancestors}
+To merge the full chain, use
+> @{pr.repository.project_id.fp_github_name} r+
+{footer}","Comment when a forward port was created and is the last of a sequence (target the limit branch).
+
+pr: the new forward port
+containing: label changing depending whether there are ancestors to merge
+ancestors: markdown formatted list of parent PRs which can be approved as part of the chain
+footer: a footer"
+runbot_merge.forwardport.intermediate,"This PR targets {pr.target.name} and is part of the forward-port chain. Further PRs will be created up to {pr.limit_id.name}.
+{footer}","Comment when a forward port was succcessfully created but is not the last of the line.
+
+pr: the new forward port
+footer: a footer"
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 76ebb427..8e7cc1f5 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -16,6 +16,7 @@ import time
 
 from difflib import Differ
 from itertools import takewhile
+from typing import Optional
 
 import requests
 import werkzeug
@@ -111,15 +112,17 @@ All substitutions are tentatively applied sequentially to the input.
         # fetch PR object and handle as *opened*
         issue, pr = gh.pr(number)
 
-        feedback = self.env['runbot_merge.pull_requests.feedback'].create
         if not self.project_id._has_branch(pr['base']['ref']):
             _logger.info("Tasked with loading PR %d for un-managed branch %s:%s, ignoring",
                          number, self.name, pr['base']['ref'])
-            feedback({
-                'repository': self.id,
-                'pull_request': number,
-                'message': "Branch `{}` is not within my remit, imma just ignore it.".format(pr['base']['ref']),
-            })
+            self.env.ref('runbot_merge.pr.load.unmanaged')._send(
+                repository=self,
+                pull_request=number,
+                format_args = {
+                    'pr': pr,
+                    'repository': self,
+                },
+            )
             return
 
         # if the PR is already loaded, force sync a few attributes
@@ -150,20 +153,13 @@ All substitutions are tentatively applied sequentially to the input.
                     'pull_request': pr,
                     'sender': {'login': self.project_id.github_prefix}
                 }) + '. '
-            feedback({
+            self.env['runbot_merge.pull_requests.feedback'].create({
                 'repository': pr_id.repository.id,
                 'pull_request': number,
                 'message': f"{edit}. {edit2}{sync}.",
             })
             return
 
-        feedback({
-            'repository': self.id,
-            'pull_request': number,
-            'message': "%sI didn't know about this PR and had to retrieve "
-                       "its information, you may have to re-approve it as "
-                       "I didn't see previous commands." % pr_id.ping()
-        })
         sender = {'login': self.project_id.github_prefix}
         # init the PR to the null commit so we can later synchronise it back
         # back to the "proper" head while resetting reviews
@@ -212,14 +208,21 @@ All substitutions are tentatively applied sequentially to the input.
             'pull_request': pr,
             'sender': sender,
         })
+        pr_id = self.env['runbot_merge.pull_requests'].search([
+            ('repository.name', '=', pr['base']['repo']['full_name']),
+            ('number', '=', number),
+        ])
         if pr['state'] == 'closed':
             # don't go through controller because try_closing does weird things
             # for safety / race condition reasons which ends up committing
             # and breaks everything
-            self.env['runbot_merge.pull_requests'].search([
-                ('repository.name', '=', pr['base']['repo']['full_name']),
-                ('number', '=', number),
-            ]).state = 'closed'
+            pr_id.state = 'closed'
+
+        self.env.ref('runbot_merge.pr.load.fetched')._send(
+            repository=self,
+            pull_request=number,
+            format_args={'pr': pr_id},
+        )
 
     def having_branch(self, branch):
         branches = self.env['runbot_merge.branch'].search
@@ -281,10 +284,11 @@ class Branch(models.Model):
                 "Target branch deactivated by %r.",
                 self.env.user.login,
             )
+            tmpl = self.env.ref('runbot_merge.pr.branch.disabled')
             self.env['runbot_merge.pull_requests.feedback'].create([{
                 'repository': pr.repository.id,
                 'pull_request': pr.number,
-                'message': f'Hey {pr.ping()}the target branch {pr.target.name!r} has been disabled, you may want to close this PR.',
+                'message': tmpl._format(pr=pr),
             } for pr in self.prs])
         return True
 
@@ -392,11 +396,11 @@ class Branch(models.Model):
                         reason = json.loads(str(reason))['message'].lower()
 
                     pr.state = 'error'
-                    self.env['runbot_merge.pull_requests.feedback'].create({
-                        'repository': pr.repository.id,
-                        'pull_request': pr.number,
-                        'message': f'{pr.ping()}unable to stage: {reason}',
-                    })
+                    self.env.ref('runbot_merge.pr.merge.failed')._send(
+                        repository=pr.repository,
+                        pull_request=pr.number,
+                        format_args= {'pr': pr, 'reason': reason, 'exc': e},
+                    )
             else:
                 first = False
 
@@ -584,16 +588,16 @@ class PullRequests(models.Model):
     repo_name = fields.Char(related='repository.name')
     message_title = fields.Char(compute='_compute_message_title')
 
-    def ping(self, author=True, reviewer=True):
-        P = self.env['res.partner']
-        s = ' '.join(
-            f'@{p.github_login}'
-            for p in (self.author if author else P) | (self.reviewed_by if reviewer else P)
-            if p
-        )
-        if s:
-            s += ' '
-        return s
+    ping = fields.Char(compute='_compute_ping')
+
+    @api.depends('author.github_login', 'reviewed_by.github_login')
+    def _compute_ping(self):
+        for pr in self:
+            s = ' '.join(
+                f'@{p.github_login}'
+                for p in (pr.author | pr.reviewed_by )
+            )
+            pr.ping = s and (s + ' ')
 
     @api.depends('repository.name', 'number')
     def _compute_url(self):
@@ -739,11 +743,11 @@ class PullRequests(models.Model):
             return
 
         if target and not repo.project_id._has_branch(target):
-            self.env['runbot_merge.pull_requests.feedback'].create({
-                'repository': repo.id,
-                'pull_request': number,
-                'message': "I'm sorry. Branch `{}` is not within my remit.".format(target),
-            })
+            self.env.ref('runbot_merge.pr.fetch.unmanaged')._send(
+                repository=repo,
+                pull_request=number,
+                format_args={'repository': repo, 'branch': target, 'number': number}
+            )
             return
 
         pr = self.search([
@@ -825,16 +829,15 @@ class PullRequests(models.Model):
             )
             return 'ok'
 
-        Feedback = self.env['runbot_merge.pull_requests.feedback']
         if not (is_author or any(cmd == 'override' for cmd, _ in commands)):
             # no point even parsing commands
             _logger.info("ignoring comment of %s (%s): no ACL to %s",
                           login, name, self.display_name)
-            Feedback.create({
-                'repository': self.repository.id,
-                'pull_request': self.number,
-                'message': "I'm sorry, @{}. I'm afraid I can't do that.".format(login)
-            })
+            self.env.ref('runbot_merge.command.access.no')._send(
+                repository=self.repository,
+                pull_request=self.number,
+                format_args={'user': login, 'pr': self}
+            )
             return 'ignored'
 
         applied, ignored = [], []
@@ -890,11 +893,11 @@ class PullRequests(models.Model):
                     if self.status == 'failure':
                         # the normal infrastructure is for failure and
                         # prefixes messages with "I'm sorry"
-                        Feedback.create({
-                            'repository': self.repository.id,
-                            'pull_request': self.number,
-                            'message': "@{} you may want to rebuild or fix this PR as it has failed CI.".format(login),
-                        })
+                        self.env.ref("runbot_merge.command.approve.failure")._send(
+                            repository=self.repository,
+                            pull_request=self.number,
+                            format_args={'user': login, 'pr': self},
+                        )
                 elif not param and is_author:
                     newstate = RMINUS.get(self.state)
                     if self.priority == 0 or newstate:
@@ -902,11 +905,11 @@ class PullRequests(models.Model):
                             self.state = newstate
                         if self.priority == 0:
                             self.priority = 1
-                            Feedback.create({
-                                'repository': self.repository.id,
-                                'pull_request': self.number,
-                                'message': "PR priority reset to 1, as pull requests with priority 0 ignore review state.",
-                            })
+                            self.env.ref("runbot_merge.command.unapprove.p0")._send(
+                                repository=self.repository,
+                                pull_request=self.number,
+                                format_args={'user': login, 'pr': self},
+                            )
                         self.unstage("unreviewed (r-) by %s", login)
                         ok = True
                     else:
@@ -938,11 +941,11 @@ class PullRequests(models.Model):
                     self.merge_method = param
                     ok = True
                     explanation = next(label for value, label in type(self).merge_method.selection if value == param)
-                    Feedback.create({
-                        'repository': self.repository.id,
-                        'pull_request': self.number,
-                        'message':"Merge method set to %s." % explanation
-                    })
+                    self.env.ref("runbot_merge.command.method")._send(
+                        repository=self.repository,
+                        pull_request=self.number,
+                        format_args={'new_method': explanation, 'pr': self, 'user': login},
+                    )
             elif command == 'override':
                 overridable = author.override_rights\
                     .filtered(lambda r: not r.repository_id or (r.repository_id == self.repository))\
@@ -983,7 +986,7 @@ class PullRequests(models.Model):
         if msgs:
             joiner = ' ' if len(msgs) == 1 else '\n- '
             msgs.insert(0, "I'm sorry, @{}:".format(login))
-            Feedback.create({
+            self.env['runbot_merge.pull_requests.feedback'].create({
                 'repository': self.repository.id,
                 'pull_request': self.number,
                 'message': joiner.join(msgs),
@@ -1078,11 +1081,11 @@ class PullRequests(models.Model):
     def _notify_ci_failed(self, ci):
         # only report an issue of the PR is already approved (r+'d)
         if self.state == 'approved':
-            self.env['runbot_merge.pull_requests.feedback'].create({
-                'repository': self.repository.id,
-                'pull_request': self.number,
-                'message': "%s%r failed on this reviewed PR." % (self.ping(), ci),
-            })
+            self.env.ref("runbot_merge.failure.approved")._send(
+                repository=self.repository,
+                pull_request=self.number,
+                format_args={'pr': self, 'status': ci}
+            )
 
     def _auto_init(self):
         super(PullRequests, self)._auto_init()
@@ -1108,11 +1111,11 @@ class PullRequests(models.Model):
         pr._validate(json.loads(c.statuses or '{}'))
 
         if pr.state not in ('closed', 'merged'):
-            self.env['runbot_merge.pull_requests.feedback'].create({
-                'repository': pr.repository.id,
-                'pull_request': pr.number,
-                'message': f"[Pull request status dashboard]({pr.url}).",
-            })
+            self.env.ref('runbot_merge.pr.created')._send(
+                repository=pr.repository,
+                pull_request=pr.number,
+                format_args={'pr': pr},
+            )
         return pr
 
     def _from_gh(self, description, author=None, branch=None, repo=None):
@@ -1211,14 +1214,14 @@ class PullRequests(models.Model):
             unready = (prs - ready).sorted(key=lambda p: (p.repository.name, p.number))
 
             for r in ready:
-                self.env['runbot_merge.pull_requests.feedback'].create({
-                    'repository': r.repository.id,
-                    'pull_request': r.number,
-                    'message': "{}linked pull request(s) {} not ready. Linked PRs are not staged until all of them are ready.".format(
-                        r.ping(),
-                        ', '.join(map('{0.display_name}'.format, unready))
-                    )
-                })
+                self.env.ref('runbot_merge.pr.linked.not_ready')._send(
+                    repository=r.repository,
+                    pull_request=r.number,
+                    format_args={
+                        'pr': r,
+                        'siblings': ', '.join(map('{0.display_name}'.format, unready))
+                    },
+                )
                 r.link_warned = True
                 if commit:
                     self.env.cr.commit()
@@ -1236,14 +1239,11 @@ class PullRequests(models.Model):
             ('merge_method', '=', False),
             ('method_warned', '=', False),
         ]):
-            self.env['runbot_merge.pull_requests.feedback'].create({
-                'repository': r.repository.id,
-                'pull_request': r.number,
-                'message': "%sbecause this PR has multiple commits, I need to know how to merge it:\n\n%s" % (
-                    r.ping(),
-                    methods,
-                )
-            })
+            self.env.ref('runbot_merge.pr.merge_method')._send(
+                repository=r.repository,
+                pull_request=r.number,
+                format_args={'pr': r, 'methods':methods},
+            )
             r.method_warned = True
             if commit:
                 self.env.cr.commit()
@@ -1663,6 +1663,33 @@ class Feedback(models.Model):
                 to_remove.append(f.id)
         self.browse(to_remove).unlink()
 
+class FeedbackTemplate(models.Model):
+    _name = 'runbot_merge.pull_requests.feedback.template'
+    _description = "str.format templates for feedback messages, no integration," \
+                   "but that's their purpose"
+    _inherit = ['mail.thread']
+
+    template = fields.Text(tracking=True)
+    help = fields.Text(readonly=True)
+
+    def _format(self, **args):
+        return self.template.format_map(args)
+
+    def _send(self, *, repository: Repository, pull_request: int, format_args: dict, token_field: Optional[str] = None) -> Optional[Feedback]:
+        try:
+            feedback = {
+                'repository': repository.id,
+                'pull_request': pull_request,
+                'message': self.template.format_map(format_args),
+            }
+            if token_field:
+                feedback['token_field'] = token_field
+            return self.env['runbot_merge.pull_requests.feedback'].create(feedback)
+        except Exception:
+            _logger.exception("Failed to render template %s", self.get_external_id())
+            raise
+            return None
+
 class Commit(models.Model):
     """Represents a commit onto which statuses might be posted,
     independent of everything else as commits can be created by
@@ -1900,11 +1927,11 @@ class Stagings(models.Model):
         prs = prs or self.batch_ids.prs
         prs.write({'state': 'error'})
         for pr in prs:
-            self.env['runbot_merge.pull_requests.feedback'].create({
-                'repository': pr.repository.id,
-                'pull_request': pr.number,
-                'message': "%sstaging failed: %s" % (pr.ping(), message),
-            })
+           self.env.ref('runbot_merge.pr.staging.fail')._send(
+               repository=pr.repository,
+               pull_request=pr.number,
+               format_args={'pr': pr, 'message': message},
+           )
 
         self.batch_ids.write({'active': False})
         self.write({
@@ -2213,31 +2240,16 @@ class Batch(models.Model):
                     "data mismatch on %s:\n%s",
                     pr.display_name, diff
                 )
-                self.env['runbot_merge.pull_requests.feedback'].create({
-                    'repository': pr.repository.id,
-                    'pull_request': pr.number,
-                    'message': """\
-{ping}we apparently missed updates to this PR and tried to stage it in a state \
-which might not have been approved.
-
-The properties {mismatch} were not correctly synchronized and have been updated.
-
-<details><summary>differences</summary>
-
-```diff
-{diff}```
-</details>
-
-Note that we are unable to check the properties {unchecked}.
-
-Please check and re-approve.
-""".format(
-    ping=pr.ping(),
-    mismatch=', '.join(pr_fields[f].string for f in e.args[0]),
-    diff=diff,
-    unchecked=', '.join(pr_fields[f].string for f in UNCHECKABLE),
-)
-                })
+                self.env.ref('runbot_merge.pr.staging.mismatch')._send(
+                    repository=pr.repository,
+                    pull_request=pr.number,
+                    format_args={
+                        'pr': pr,
+                        'mismatch': ', '.join(pr_fields[f].string for f in e.args[0]),
+                        'diff': diff,
+                        'unchecked': ', '.join(pr_fields[f].string for f in UNCHECKABLE)
+                    }
+                )
                 return self.env['runbot_merge.batch']
 
         # update meta to new heads
diff --git a/runbot_merge/security/ir.model.access.csv b/runbot_merge/security/ir.model.access.csv
index a6cb79ba..006f59de 100644
--- a/runbot_merge/security/ir.model.access.csv
+++ b/runbot_merge/security/ir.model.access.csv
@@ -25,3 +25,4 @@ access_runbot_merge_pull_requests,User access to PR,model_runbot_merge_pull_requ
 access_runbot_merge_pull_requests_feedback,Users have no reason to access feedback,model_runbot_merge_pull_requests_feedback,,0,0,0,0
 access_runbot_merge_review_rights_2,Users can see partners,model_res_partner_review,base.group_user,1,0,0,0
 access_runbot_merge_review_override_2,Users can see partners,model_res_partner_override,base.group_user,1,0,0,0
+runbot_merge.access_runbot_merge_pull_requests_feedback_template,access_runbot_merge_pull_requests_feedback_template,runbot_merge.model_runbot_merge_pull_requests_feedback_template,base.group_system,1,1,0,0
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index c15925b1..305a79e2 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -3212,10 +3212,10 @@ class TestUnknownPR:
             seen(env, prx, users),
             (users['reviewer'], 'hansen r+'),
             (users['reviewer'], 'hansen r+'),
-            (users['user'], "I didn't know about this PR and had to "
+            seen(env, prx, users),
+            (users['user'], f"@{users['user']} @{users['reviewer']} I didn't know about this PR and had to "
                             "retrieve its information, you may have to "
                             "re-approve it as I didn't see previous commands."),
-            seen(env, prx, users),
         ]
 
         pr = env['runbot_merge.pull_requests'].search([
@@ -3265,10 +3265,13 @@ class TestUnknownPR:
         assert pr.comments == [
             seen(env, pr, users),
             (users['reviewer'], 'hansen r+'),
-            (users['user'], "I didn't know about this PR and had to retrieve "
+            seen(env, pr, users),
+            # reviewer is set because fetch replays all the comments (thus
+            # setting r+ and reviewer) but then syncs the head commit thus
+            # unsetting r+ but leaving the reviewer
+            (users['user'], f"@{users['user']} @{users['reviewer']} I didn't know about this PR and had to retrieve "
                             "its information, you may have to re-approve it "
                             "as I didn't see previous commands."),
-            seen(env, pr, users),
         ]
 
     def test_rplus_unmanaged(self, env, repo, users, config):
diff --git a/runbot_merge/tests/test_disabled_branch.py b/runbot_merge/tests/test_disabled_branch.py
index f56ea125..2cff73ce 100644
--- a/runbot_merge/tests/test_disabled_branch.py
+++ b/runbot_merge/tests/test_disabled_branch.py
@@ -63,7 +63,7 @@ def test_existing_pr_disabled_branch(env, project, make_repo, setreviewers, conf
     assert pr.comments == [
         (users['reviewer'], "hansen r+"),
         seen(env, pr, users),
-        (users['user'], "Hey @%(user)s @%(reviewer)s the target branch 'other' has been disabled, you may want to close this PR." % users),
+        (users['user'], "@%(user)s @%(reviewer)s the target branch 'other' has been disabled, you may want to close this PR." % users),
     ]
 
     with repo:
diff --git a/runbot_merge/views/configuration.xml b/runbot_merge/views/configuration.xml
index 70e8d710..32d3f3e7 100644
--- a/runbot_merge/views/configuration.xml
+++ b/runbot_merge/views/configuration.xml
@@ -33,11 +33,42 @@
         </field>
     </record>
 
-    <menuitem name="Configuration" id="menu_configuration" parent="runbot_merge_menu"/>
+    <record id="action_feedback" model="ir.actions.act_window">
+        <field name="name">Feedback Templates tree</field>
+        <field name="res_model">runbot_merge.pull_requests.feedback.template</field>
+    </record>
+    <record id="tree_feedback" model="ir.ui.view">
+        <field name="name">Feedback Templates</field>
+        <field name="model">runbot_merge.pull_requests.feedback.template</field>
+        <field name="arch" type="xml">
+            <tree>
+                <field name="template"/>
+                <field name="help"/>
+            </tree>
+        </field>
+    </record>
+    <record id="form_feedback" model="ir.ui.view">
+        <field name="name">Feedback Templates form</field>
+        <field name="model">runbot_merge.pull_requests.feedback.template</field>
+        <field name="arch" type="xml">
+            <form>
+                <sheet>
+                    <field name="help"/>
+                    <field name="template"/>
+                </sheet>
+                <div class="oe_chatter">
+                    <field name="message_ids"/>
+                </div>
+            </form>
+        </field>
+    </record>
+
+    <menuitem name="Configuration" id="menu_configuration" parent="runbot_merge_menu">
         <menuitem name="CI Overrides" id="menu_configuration_overrides"
-                  parent="menu_configuration"
                   action="action_overrides"/>
         <menuitem name="Review Rights" id="menu_configuration_review"
-                  parent="menu_configuration"
                   action="action_review"/>
+        <menuitem name="Feedback Templates" id="menu_configuration_feedback"
+                  action="action_feedback"/>
+    </menuitem>
 </odoo>

From cd4ded899b56ded376824a117bfaa444b39f8c92 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 13 Jun 2023 14:22:40 +0200
Subject: [PATCH 009/192] [IMP] runbot_merge: error reporting

Largely informed by sentry,

- Fix an inconsistency in staging ref verification, `set_ref`
  internally waits for the observed head to match the requested head,
  but then `try_staging` would re-check that and immediately fail if
  it didn't.

  Because github is *eventually* consistent (hopefully) this second
  check can fail (and is also an extra API call), breaking staging
  unnecessarily, especially as we're still going to wait for the
  update to be visible to git.

  Remove this redundant check entirely, as github provides no way to
  ensure we have a consistent view of anything, it doesn't have much
  value and can do much harm.
- Add github request id to one of the sanity check warnings as that
  could be a useful thing to send upstream, missing github request ids
  in the future should be noted and added.
- Reworked the GH object's calls to be clearer and more coherent:
  consistently log the same thing on all GH errors (if `check`),
  rather than just on the one without a `check` entry.

  Also remove `raise_for_status` and raise `HTTPError` by hand every
  time we hit a status >= 400, so we always forward the response body
  no matter what its type is.
- Try again to log the request body (in full as it should be pretty
  small), also remove stripping since we specifically wanted to add a
  newline at the start, I've no idea what I was thinking.

Fixes #735, #764, #544
---
 runbot_merge/github.py               | 60 ++++++++++++++--------------
 runbot_merge/models/pull_requests.py |  7 ----
 2 files changed, 31 insertions(+), 36 deletions(-)

diff --git a/runbot_merge/github.py b/runbot_merge/github.py
index 27cf6305..4f2b6fc8 100644
--- a/runbot_merge/github.py
+++ b/runbot_merge/github.py
@@ -1,12 +1,11 @@
 import collections.abc
 import itertools
-import json as json_
+import json
 import logging
 import logging.handlers
 import os
 import pathlib
 import pprint
-import textwrap
 import unicodedata
 
 import requests
@@ -47,7 +46,7 @@ def _init_gh_logger():
 if odoo.netsvc._logger_init:
     _init_gh_logger()
 
-GH_LOG_PATTERN = """=> {method} /{self._repo}/{path}{qs}{body}
+GH_LOG_PATTERN = """=> {method} /{path}{qs}{body}
 
 <= {r.status_code} {r.reason}
 {headers}
@@ -62,7 +61,7 @@ class GH(object):
         session.headers['Authorization'] = 'token {}'.format(token)
         session.headers['Accept'] = 'application/vnd.github.symmetra-preview+json'
 
-    def _log_gh(self, logger, method, path, params, json, response, level=logging.INFO):
+    def _log_gh(self, logger: logging.Logger, response: requests.Response, level: int = logging.INFO, extra=None):
         """ Logs a pair of request / response to github, to the specified
         logger, at the specified level.
 
@@ -70,11 +69,14 @@ class GH(object):
         bodies, at least in part) so we have as much information as possible
         for post-mortems.
         """
-        body = body2 = ''
+        req = response.request
+        url = werkzeug.urls.url_parse(req.url)
+        if url.netloc is not 'api.github.com':
+            return
 
-        if json:
-            body = '\n' + textwrap.indent('\t', pprint.pformat(json, indent=4))
+        body = '' if not req.body else ('\n' + pprint.pformat(json.loads(req.body.decode()), indent=4))
 
+        body2 = ''
         if response.content:
             if _is_json(response):
                 body2 = pprint.pformat(response.json(), depth=4)
@@ -87,19 +89,15 @@ class GH(object):
                 )
 
         logger.log(level, GH_LOG_PATTERN.format(
-            self=self,
             # requests data
-            method=method, path=path,
-            qs='' if not params else ('?' + werkzeug.urls.url_encode(params)),
-            body=utils.shorten(body.strip(), 400),
+            method=req.method, path=url.path, qs=url.query, body=body,
             # response data
             r=response,
             headers='\n'.join(
                 '\t%s: %s' % (h, v) for h, v in response.headers.items()
             ),
             body2=utils.shorten(body2.strip(), 400)
-        ))
-        return body2
+        ), extra=extra)
 
     def __call__(self, method, path, params=None, json=None, check=True):
         """
@@ -107,21 +105,21 @@ class GH(object):
         """
         path = f'/repos/{self._repo}/{path}'
         r = self._session.request(method, self._url + path, params=params, json=json)
-        self._log_gh(_gh, method, path, params, json, r)
+        self._log_gh(_gh, r)
         if check:
-            if isinstance(check, collections.abc.Mapping):
-                exc = check.get(r.status_code)
-                if exc:
-                    raise exc(r.text)
-            if r.status_code >= 400:
-                body = self._log_gh(
-                    _logger, method, path, params, json, r, level=logging.ERROR)
-                if not isinstance(body, (bytes, str)):
-                    raise requests.HTTPError(
-                        json_.dumps(body, indent=4),
-                        response=r
-                    )
-            r.raise_for_status()
+            try:
+                if isinstance(check, collections.abc.Mapping):
+                    exc = check.get(r.status_code)
+                    if exc:
+                        raise exc(r.text)
+                if r.status_code >= 400:
+                    raise requests.HTTPError(r.text, response=r)
+            except Exception:
+                self._log_gh(_logger, r, level=logging.ERROR, extra={
+                    'github-request-id': r.headers.get('x-github-request-id'),
+                })
+                raise
+
         return r
 
     def user(self, username):
@@ -180,13 +178,17 @@ class GH(object):
         if r.status_code == 200:
             head = r.json()['object']['sha']
         else:
-            head = '<Response [%s]: %s)>' % (r.status_code, r.json() if _is_json(r) else r.text)
+            head = '<Response [%s]: %s)>' % (r.status_code, r.text)
 
         if head == to:
             _logger.debug("Sanity check ref update of %s to %s: ok", branch, to)
             return
 
-        _logger.warning("Sanity check ref update of %s, expected %s got %s", branch, to, head)
+        _logger.warning(
+            "Sanity check ref update of %s, expected %s got %s (response-id %s)",
+            branch, to, head,
+            r.headers.get('x-github-request-id')
+        )
         return head
 
     def fast_forward(self, branch, sha):
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 8e7cc1f5..a25b3cd2 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -463,13 +463,6 @@ For-Commit-Id: %s
             )
             refname = 'staging.{}'.format(self.name)
             it['gh'].set_ref(refname, staging_head)
-            # asserts that the new head is visible through the api
-            head = it['gh'].head(refname)
-            assert head == staging_head,\
-                "[api] updated %s:%s to %s but found %s" % (
-                    r.name, refname,
-                    staging_head, head,
-                )
 
             i = itertools.count()
             @utils.backoff(delays=WAIT_FOR_VISIBILITY, exc=TimeoutError)

From 06a3a1bab57ad4cf72409d4823776d4e43da2195 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 14 Jun 2023 13:34:22 +0200
Subject: [PATCH 010/192] [IMP] runbot_merge: add sentry filtering, rework some
 error messages

- move sentry configuration and add exception-based filtering
- clarify and reclassify (e.g. from warning to info) a few messages
- convert assertions in rebase to MergeError so they can be correctly
  logged & reported, and ignored by sentry, also clarify them
  (especially the consistency one)

Related to #544
---
 runbot_merge/__init__.py             | 38 +------------
 runbot_merge/controllers/__init__.py | 16 +++---
 runbot_merge/github.py               | 22 +++++---
 runbot_merge/models/pull_requests.py | 13 ++---
 runbot_merge/sentry.py               | 80 ++++++++++++++++++++++++++++
 5 files changed, 110 insertions(+), 59 deletions(-)
 create mode 100644 runbot_merge/sentry.py

diff --git a/runbot_merge/__init__.py b/runbot_merge/__init__.py
index f4821304..31b308ad 100644
--- a/runbot_merge/__init__.py
+++ b/runbot_merge/__init__.py
@@ -1,41 +1,5 @@
-import logging
-from os import environ
-
-import sentry_sdk
-from sentry_sdk.integrations.logging import LoggingIntegration
-from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware
-
-from odoo import http
 from . import models, controllers
-
-def delegate(self, attr):
-    return getattr(self.app, attr)
-SentryWsgiMiddleware.__getattr__ = delegate
-
-def enable_sentry():
-    logger = logging.getLogger('runbot_merge')
-
-    dsn = environ.get('SENTRY_DSN')
-    if not dsn:
-        logger.info("No DSN found, skipping sentry...")
-        return
-
-    try:
-        sentry_sdk.init(
-            dsn,
-            integrations=[
-                # note: if the colorformatter is enabled, sentry gets lost
-                # and classifies everything as errors because it fails to
-                # properly classify levels as the colorformatter injects
-                # the ANSI color codes right into LogRecord.levelname
-                LoggingIntegration(level=logging.INFO, event_level=logging.WARNING),
-            ]
-        )
-        http.root = SentryWsgiMiddleware(http.root)
-    except Exception:
-        logger.exception("DSN found, failed to enable sentry...")
-    else:
-        logger.info("DSN found, sentry enabled...")
+from .sentry import enable_sentry
 
 def _check_citext(cr):
     cr.execute("select 1 from pg_extension where extname = 'citext'")
diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 51d8c71d..4781207d 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -235,14 +235,14 @@ def handle_pr(env, event):
                 oldstate,
             )
             return 'Closed {}'.format(pr_obj.display_name)
-        else:
-            _logger.warning(
-                '%s tried to close %s (state=%s)',
-                event['sender']['login'],
-                pr_obj.display_name,
-                oldstate,
-            )
-            return 'Ignored: could not lock rows (probably being merged)'
+
+        _logger.info(
+            '%s tried to close %s (state=%s) but locking failed',
+            event['sender']['login'],
+            pr_obj.display_name,
+            oldstate,
+        )
+        return 'Ignored: could not lock rows (probably being merged)'
 
     if event['action'] == 'reopened' :
         if pr_obj.state == 'merged':
diff --git a/runbot_merge/github.py b/runbot_merge/github.py
index 4f2b6fc8..e5e5fa9e 100644
--- a/runbot_merge/github.py
+++ b/runbot_merge/github.py
@@ -275,7 +275,7 @@ class GH(object):
         try:
             r = r.json()
         except Exception:
-            raise MergeError("Got non-JSON reponse from github: %s %s (%s)" % (r.status_code, r.reason, r.text))
+            raise MergeError("got non-JSON reponse from github: %s %s (%s)" % (r.status_code, r.reason, r.text))
         _logger.debug(
             "merge(%s, %s (%s), %s) -> %s",
             self._repo, dest, r['parents'][0]['sha'],
@@ -297,10 +297,17 @@ class GH(object):
         logger.debug("rebasing %s, %s on %s (reset=%s, commits=%s)",
                      self._repo, pr, dest, reset, len(commits))
 
-        assert commits, "can't rebase a PR with no commits"
+        if not commits:
+            raise MergeError("PR has no commits")
         prev = original_head
         for original in commits:
-            assert len(original['parents']) == 1, "can't rebase commits with more than one parent"
+            if len(original['parents']) != 1:
+                raise MergeError(
+                    "commits with multiple parents ({sha}) can not be rebased, "
+                    "either fix the branch to remove merges or merge without "
+                    "rebasing".format_map(
+                    original
+                ))
             tmp_msg = 'temp rebasing PR %s (%s)' % (pr, original['sha'])
             merged = self.merge(original['sha'], dest, tmp_msg)
 
@@ -309,9 +316,12 @@ class GH(object):
             # expect (either original_head or the previously merged commit)
             [base_commit] = (parent['sha'] for parent in merged['parents']
                              if parent['sha'] != original['sha'])
-            assert prev == base_commit,\
-                "Inconsistent view of %s between head (%s) and merge (%s)" % (
-                    dest, prev, base_commit,
+            if prev != base_commit:
+                raise MergeError(
+                    f"Inconsistent view of branch {dest} while rebasing "
+                    f"PR {pr} expected commit {prev} but the other parent of "
+                    f"merge commit {merged['sha']} is {base_commit}.\n\n"
+                    f"The branch may be getting concurrently modified."
                 )
             prev = merged['sha']
             original['new_tree'] = merged['tree']['sha']
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index a25b3cd2..c2ccb47a 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -388,7 +388,7 @@ class Branch(models.Model):
                     if len(e.args) > 1 and e.args[1]:
                         reason = e.args[1]
                     else:
-                        reason = e.__context__
+                        reason = e.__cause__ or e.__context__
                     # if the reason is a json document, assume it's a github
                     # error and try to extract the error message to give it to
                     # the user
@@ -1588,7 +1588,7 @@ class Tagging(models.Model):
             try:
                 gh.change_tags(pr, tags_remove, tags_add)
             except Exception:
-                _logger.exception(
+                _logger.info(
                     "Error while trying to change the tags of %s#%s from %s to %s",
                     repo.name, pr, remove, add,
                 )
@@ -2211,8 +2211,8 @@ class Batch(models.Model):
                         it = meta[to_revert.repository]
                         it['gh'].set_ref('tmp.{}'.format(to_revert.target.name), it['head'])
                     raise
-            except github.MergeError:
-                raise exceptions.MergeError(pr)
+            except github.MergeError as e:
+                raise exceptions.MergeError(pr) from e
             except exceptions.Mismatch as e:
                 def format_items(items):
                     """ Bit of a pain in the ass because difflib really wants
@@ -2229,10 +2229,7 @@ class Batch(models.Model):
                 old = list(format_items((n, str(v)) for n, v, _ in e.args[1]))
                 new = list(format_items((n, str(v)) for n, _, v in e.args[1]))
                 diff = ''.join(Differ().compare(old, new))
-                _logger.warning(
-                    "data mismatch on %s:\n%s",
-                    pr.display_name, diff
-                )
+                _logger.info("data mismatch on %s:\n%s", pr.display_name, diff)
                 self.env.ref('runbot_merge.pr.staging.mismatch')._send(
                     repository=pr.repository,
                     pull_request=pr.number,
diff --git a/runbot_merge/sentry.py b/runbot_merge/sentry.py
new file mode 100644
index 00000000..55f3214c
--- /dev/null
+++ b/runbot_merge/sentry.py
@@ -0,0 +1,80 @@
+import logging
+from os import environ
+
+import sentry_sdk
+from sentry_sdk.integrations.logging import LoggingIntegration
+from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware
+
+from odoo import http
+from runbot_merge.exceptions import FastForwardError, Mismatch, MergeError, Unmergeable
+
+
+def delegate(self, attr):
+    return getattr(self.app, attr)
+SentryWsgiMiddleware.__getattr__ = delegate
+
+def enable_sentry():
+    logger = logging.getLogger('runbot_merge')
+
+    dsn = environ.get('SENTRY_DSN')
+    if not dsn:
+        logger.info("No DSN found, skipping sentry...")
+        return
+
+    try:
+        setup_sentry(dsn)
+    except Exception:
+        logger.exception("DSN found, failed to enable sentry...")
+    else:
+        logger.info("DSN found, sentry enabled...")
+
+
+def setup_sentry(dsn):
+    sentry_sdk.init(
+        dsn,
+        traces_sample_rate=1.0,
+        integrations=[
+            # note: if the colorformatter is enabled, sentry gets lost
+            # and classifies everything as errors because it fails to
+            # properly classify levels as the colorformatter injects
+            # the ANSI color codes right into LogRecord.levelname
+            LoggingIntegration(level=logging.INFO, event_level=logging.WARNING),
+        ],
+        before_send=event_filter,
+    )
+    http.root = SentryWsgiMiddleware(http.root)
+
+dummy_record = logging.LogRecord(name="", level=logging.NOTSET, pathname='', lineno=0, msg='', args=(), exc_info=None)
+# mapping of exception types to predicates, if the predicate returns `True` the
+# exception event should be suppressed
+SUPPRESS_EXCEPTION = {
+    # Someone else deciding to push directly to the branch (which is generally
+    # what leads to this error) is not really actionable.
+    #
+    # Other possibilities are more structural and thus we probably want to know:
+    # - other 422 Unprocessable github errors (likely config issues):
+    #   - reference does not exist
+    #   - object does not exist
+    #   - object is not a commit
+    #   - branch protection issue
+    # - timeout on ref update (github probably dying)
+    # - other HTTP error (also github probably dying)
+    #
+    # might be worth using richer exceptions to make this clearer, and easier to classify
+    FastForwardError: lambda e: 'not a fast forward' in str(e.__cause__),
+    # Git conflict when merging (or non-json response which is weird),
+    # notified on PR
+    MergeError: lambda _: True,
+    # Failed preconditions on merging, notified on PR
+    Unmergeable: lambda _: True,
+}
+def event_filter(event, hint):
+    # event['level'], event['logger'], event['logentry'], event['exception']
+    # known hints: log_record: LogRecord, exc_info: (type, BaseExeption, Traceback) | None
+    exc_info = hint.get('exc_info') or hint.get('log_record', dummy_record).exc_info
+    if exc_info:
+        etype, exc, _ = exc_info
+        if SUPPRESS_EXCEPTION.get(etype, lambda _: False)(exc):
+            return None
+
+

From ed0fd88854be57826fe8463335338de5946a59a9 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 15 Jun 2023 08:22:38 +0200
Subject: [PATCH 011/192] [ADD] runbot_merge: sentry instrumentation

Currently sentry is only hooked from the outside, which doesn't
necessarily provide sufficiently actionable information.

Add some a few hooks to (try and) report odoo / mergebot metadata:

- add the user to WSGI transactions
- add a transaction (with users) around crons
- add the webhook event info to webhook requests
- add a few spans to the long-running crons, when they cover multiple
  units per iteration (e.g. a span per branch being staged)

Closes #544
---
 forwardport/models/forwardport.py    |  8 +++++-
 runbot_merge/controllers/__init__.py |  9 +++++++
 runbot_merge/models/project.py       |  6 ++++-
 runbot_merge/models/pull_requests.py |  6 ++++-
 runbot_merge/sentry.py               | 38 +++++++++++++++++++++++++++-
 5 files changed, 63 insertions(+), 4 deletions(-)

diff --git a/forwardport/models/forwardport.py b/forwardport/models/forwardport.py
index 78c23a46..6e3e0c24 100644
--- a/forwardport/models/forwardport.py
+++ b/forwardport/models/forwardport.py
@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
 import logging
 import pathlib
+
+import sentry_sdk
+
 import resource
 import subprocess
 import uuid
@@ -28,7 +31,8 @@ class Queue:
     def _process(self):
         for b in self.search(self._search_domain(), order='create_date, id', limit=self.limit):
             try:
-                b._process_item()
+                with sentry_sdk.start_span(description=self._name):
+                    b._process_item()
                 b.unlink()
                 self.env.cr.commit()
             except Exception:
@@ -68,6 +72,7 @@ class ForwardPortTasks(models.Model, Queue):
 
     def _process_item(self):
         batch = self.batch_id
+        sentry_sdk.set_tag('forward-porting', batch.prs.mapped('display_name'))
         newbatch = batch.prs._port_forward()
 
         if newbatch:
@@ -109,6 +114,7 @@ class UpdateQueue(models.Model, Queue):
 
     def _process_item(self):
         previous = self.new_root
+        sentry_sdk.set_tag("update-root", self.new_root.display_name)
         with ExitStack() as s:
             for child in self.new_root._iter_descendants():
                 self.env.cr.execute("""
diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 4781207d..2f9fbc20 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -3,6 +3,7 @@ import hmac
 import logging
 import json
 
+import sentry_sdk
 import werkzeug.exceptions
 
 from odoo.http import Controller, request, route
@@ -18,6 +19,13 @@ class MergebotController(Controller):
     def index(self):
         req = request.httprequest
         event = req.headers['X-Github-Event']
+        with sentry_sdk.configure_scope() as scope:
+            if scope.transaction:
+                # only in 1.8.0 (or at least 1.7.2
+                if hasattr(scope, 'set_transaction_name'):
+                    scope.set_transaction_name(f"webhook {event}")
+                else: # but our servers use 1.4.3
+                    scope.transaction = f"webhook {event}"
 
         github._gh.info(self._format(req))
 
@@ -39,6 +47,7 @@ class MergebotController(Controller):
                              req.headers.get('X-Hub-Signature'))
                 return werkzeug.exceptions.Forbidden()
 
+        sentry_sdk.set_context('webhook', request.jsonrequest)
         return c(env, request.jsonrequest)
 
     def _format(self, request):
diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index 883c91c6..8578aeb5 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -1,6 +1,8 @@
 import logging
 import re
 
+import sentry_sdk
+
 from odoo import models, fields
 
 _logger = logging.getLogger(__name__)
@@ -69,7 +71,9 @@ class Project(models.Model):
             ('staging_enabled', '=', True),
         ]):
             try:
-                with self.env.cr.savepoint():
+                with self.env.cr.savepoint(), \
+                    sentry_sdk.start_span(description=f'create staging {branch.name}') as span:
+                    span.set_tag('branch', branch.name)
                     branch.try_staging()
             except Exception:
                 _logger.exception("Failed to create staging for branch %r", branch.name)
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index c2ccb47a..19c63aef 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -19,6 +19,7 @@ from itertools import takewhile
 from typing import Optional
 
 import requests
+import sentry_sdk
 import werkzeug
 from werkzeug.datastructures import Headers
 
@@ -2031,7 +2032,10 @@ class Stagings(models.Model):
             FOR UPDATE
             ''', [tuple(self.mapped('batch_ids.prs.id'))])
             try:
-                self._safety_dance(gh, staging_heads)
+                with sentry_sdk.start_span(description="merge staging") as span:
+                    span.set_tag("staging", self.id)
+                    span.set_tag("branch", self.target.name)
+                    self._safety_dance(gh, staging_heads)
             except exceptions.FastForwardError as e:
                 logger.warning(
                     "Could not fast-forward successful staging on %s:%s",
diff --git a/runbot_merge/sentry.py b/runbot_merge/sentry.py
index 55f3214c..9c249e9e 100644
--- a/runbot_merge/sentry.py
+++ b/runbot_merge/sentry.py
@@ -6,7 +6,10 @@ from sentry_sdk.integrations.logging import LoggingIntegration
 from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware
 
 from odoo import http
-from runbot_merge.exceptions import FastForwardError, Mismatch, MergeError, Unmergeable
+from odoo.addons.base.models.ir_cron import ir_cron
+from odoo.http import WebRequest
+
+from .exceptions import FastForwardError, Mismatch, MergeError, Unmergeable
 
 
 def delegate(self, attr):
@@ -41,8 +44,41 @@ def setup_sentry(dsn):
             LoggingIntegration(level=logging.INFO, event_level=logging.WARNING),
         ],
         before_send=event_filter,
+        # apparently not in my version of the sdk
+        # functions_to_trace = []
     )
     http.root = SentryWsgiMiddleware(http.root)
+    instrument_odoo()
+
+def instrument_odoo():
+    """Monkeypatches odoo core to copy odoo metadata into sentry for more
+    informative events
+    """
+    # add user to wsgi request context
+    old_call_function = WebRequest._call_function
+    def _call_function(self, *args, **kwargs):
+        if self.uid:
+            sentry_sdk.set_user({
+                'id': self.uid,
+                'email': self.env.user.email,
+                'username': self.env.user.login,
+            })
+        else:
+            sentry_sdk.set_user({'username': '<public>'})
+        return old_call_function(self, *args, **kwargs)
+    WebRequest._call_function = _call_function
+
+    # create transaction for tracking crons, add user to that
+    old_callback = ir_cron._callback
+    def _callback(self, cron_name, server_action_id, job_id):
+        sentry_sdk.start_transaction(name=f"cron {cron_name}")
+        sentry_sdk.set_user({
+            'id': self.env.user.id,
+            'email': self.env.user.email,
+            'username': self.env.user.login,
+        })
+        return old_callback(self, cron_name, server_action_id, job_id)
+    ir_cron._callback = _callback
 
 dummy_record = logging.LogRecord(name="", level=logging.NOTSET, pathname='', lineno=0, msg='', args=(), exc_info=None)
 # mapping of exception types to predicates, if the predicate returns `True` the

From 9260384284cb58514594dd2c15babf53149a6858 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 15 Jun 2023 15:25:17 +0200
Subject: [PATCH 012/192] [FIX] runbot_merge: concurrency error in freeze
 wizard (hopefully)

During the 16.3 freeze an issue was noticed with the concurrency
safety of the freeze wizard (because it blew up, which caused a few
issues): it is possible for the cancelling of an active staging to the
master branch to fail, which causes the mergebot side of the freeze to
fail, but the github state is completed, which puts the entire thing
in a less than ideal state.

Especially with the additional issue that the branch inserter has its
own concurrency issue (which maybe I should fix): if there are
branches *being* forward-ported across the new branch, it's unable to
see them, and thus can not create the now-missing PRs.

Try to make the freeze wizard more resilient:

1. Take a lock on the master staging (if any) early on, this means if
   we can acquire it we should be able to cancel it, and it won't
   suffer a concurrency error.
2. Add the `process_updated_commits` cron to the set of locked crons,
   trying to read the log timeline it looks like the issue was commits
   being impacted on that staging while the action had started:
   REPEATABLE READ meant the freeze's transaction was unable to see
   the update from the commit statuses, therefore creating a diverging
   update when it cancelled the staging, which postgres then reported
   as a serialization error.

I'd like to relax the locking of the cron (to just FOR SHARE), but I
think it would work, per postgres:

> SELECT FOR UPDATE, and SELECT FOR SHARE commands behave the same as
> SELECT in terms of searching for target rows: they will only find
> target rows that were committed as of the transaction start
> time. However, such a target row might have already been updated (or
> deleted or locked) by another concurrent transaction by the time it
> is found. In this case, the repeatable read transaction will wait
> for the first updating transaction to commit or roll back (if it is
> still in progress). If the first updater rolls back, then its
> effects are negated and the repeatable read transaction can proceed
> with updating the originally found row. But if the first updater
> commits (and actually updated or deleted the row, not just locked
> it) then the repeatable read transaction will be rolled back with
> the message

This means it would be possible to lock the cron, and then get a
transaction error because the cron modified one of the records we're
going to hit while it was running: as far as the above is concerned
the cron's worker had "just locked" the row so it's fine to
continue. However this makes it more and more likely an error will be
hit when trying to freeze (to no issue, but still). We'll have to see
how that ends up.

Fixes #766 maybe
---
 runbot_merge/models/project_freeze/__init__.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py
index c6bc4330..b2c35d9e 100644
--- a/runbot_merge/models/project_freeze/__init__.py
+++ b/runbot_merge/models/project_freeze/__init__.py
@@ -177,7 +177,9 @@ class FreezeWizard(models.Model):
         if self.errors:
             return self.action_open()
 
-        conflict_crons = self.env.ref('runbot_merge.merge_cron') | self.env.ref('runbot_merge.staging_cron')
+        conflict_crons = self.env.ref('runbot_merge.merge_cron')\
+                       | self.env.ref('runbot_merge.staging_cron')\
+                       | self.env.ref('runbot_merge.process_updated_commits')
         # we don't want to run concurrently to the crons above, though we
         # don't need to prevent read access to them
         self.env.cr.execute(
@@ -190,6 +192,12 @@ class FreezeWizard(models.Model):
         # everything so the new branch is the second one, just after the branch
         # it "forks"
         master, rest = project_id.branch_ids[0], project_id.branch_ids[1:]
+        if self.bump_pr_ids and master.active_staging_id:
+            self.env.cr.execute(
+                'SELECT * FROM runbot_merge_stagings WHERE id = %s FOR UPDATE NOWAIT',
+                [master.active_staging_id]
+            )
+
         seq = itertools.count(start=1) # start reseq at 1
         commands = [
             (1, master.id, {'sequence': next(seq)}),

From 765281a66500a84f0bbf2e78fe79175b623c5d8f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 21 Jun 2023 14:24:58 +0200
Subject: [PATCH 013/192] [FIX] runbot_merge: make provisioning more resilient

A few cases of conflict were missing from the provisioning
handler.

They can't really be auto-fixed, so just output a warning and ignore
the entry, that way the rest of the provisioning succeeds.
---
 .../controllers/reviewer_provisioning.py      | 32 +++++++++++--
 runbot_merge/tests/test_provisioning.py       | 48 ++++++++++++-------
 2 files changed, 61 insertions(+), 19 deletions(-)

diff --git a/runbot_merge/controllers/reviewer_provisioning.py b/runbot_merge/controllers/reviewer_provisioning.py
index e0aa0fc7..35f07d24 100644
--- a/runbot_merge/controllers/reviewer_provisioning.py
+++ b/runbot_merge/controllers/reviewer_provisioning.py
@@ -35,6 +35,11 @@ class MergebotReviewerProvisioning(Controller):
         Partners = env['res.partner']
         Users = env['res.users']
 
+        existing_logins = set()
+        existing_oauth = set()
+        for u in Users.with_context(active_test=False).search([]):
+            existing_logins.add(u.login)
+            existing_oauth .add((u.oauth_provider_id.id, u.oauth_uid))
         existing_partners = Partners.with_context(active_test=False).search([
             '|', ('email', 'in', [u['email'] for u in users]),
                  ('github_login', 'in', [u['github_login'] for u in users])
@@ -76,9 +81,30 @@ class MergebotReviewerProvisioning(Controller):
                 to_activate |= current
             # entry doesn't have user -> create user
             if not current.user_ids:
-                # skip users without an email (= login) as that
-                # fails
                 if not new['email']:
+                    _logger.info(
+                        "Unable to create user for %s: no email in provisioning data",
+                        current.display_name
+                    )
+                    continue
+                if 'oauth_uid' in new:
+                    if (new['oauth_provider_id'], new['oauth_uid']) in existing_oauth:
+                        _logger.warning(
+                            "Attempted to create user with duplicate oauth uid "
+                            "%s with provider %r for provisioning entry %r. "
+                            "There is likely a duplicate partner (one version "
+                            "with email, one with github login)",
+                            new['oauth_uid'], odoo_provider.display_name, new,
+                        )
+                        continue
+                if new['email'] in existing_logins:
+                    _logger.warning(
+                        "Attempted to create user with duplicate login %s for "
+                        "provisioning entry %r. There is likely a duplicate "
+                        "partner (one version with email, one with github "
+                        "login)",
+                        new['email'], new,
+                    )
                     continue
 
                 new['login'] = new['email']
@@ -93,7 +119,7 @@ class MergebotReviewerProvisioning(Controller):
             # otherwise update user (if there is anything to update)
             user = current.user_ids
             if len(user) != 1:
-                _logger.warning("Got %d users for partner %s.", len(user), current.display_name)
+                _logger.warning("Got %d users for partner %s, updating first.", len(user), current.display_name)
                 user = user[:1]
             new.setdefault("active", True)
             update_vals = {
diff --git a/runbot_merge/tests/test_provisioning.py b/runbot_merge/tests/test_provisioning.py
index af939305..db26e5e7 100644
--- a/runbot_merge/tests/test_provisioning.py
+++ b/runbot_merge/tests/test_provisioning.py
@@ -1,4 +1,3 @@
-import pytest
 import requests
 
 GEORGE = {
@@ -30,24 +29,13 @@ def test_basic_provisioning(env, port):
     r = provision_user(port, [dict(GEORGE, name="x", github_login="y", sub="42")])
     assert r == [0, 1]
 
-    # can't fail anymore because github_login now used to look up the existing
-    # user
-    # with pytest.raises(Exception):
-    #     provision_user(port, [{
-    #         'name': "other@example.org",
-    #         'email': "x",
-    #         'github_login': "y",
-    #         'sub': "42"
-    #     }])
-
     r = provision_user(port, [dict(GEORGE, active=False)])
     assert r == [0, 1]
     assert not env['res.users'].search([('login', '=', GEORGE['email'])])
     assert env['res.partner'].search([('email', '=', GEORGE['email'])])
 
 def test_upgrade_partner(env, port):
-    # If a partner exists for a github login (and / or email?) it can be
-    # upgraded by creating a user for it
+    # matching partner with an email but no github login
     p = env['res.partner'].create({
         'name': GEORGE['name'],
         'email': GEORGE['email'],
@@ -64,6 +52,7 @@ def test_upgrade_partner(env, port):
     p.user_ids.unlink()
     p.unlink()
 
+    # matching partner with a github login but no email
     p = env['res.partner'].create({
         'name': GEORGE['name'],
         'github_login': GEORGE['github_login'],
@@ -77,14 +66,14 @@ def test_upgrade_partner(env, port):
         'email': GEORGE['email'],
     }]
 
-    # deactivate user to see if provisioning re-enables them
+    # matching partner with a deactivated user
     p.user_ids.active = False
     r = provision_user(port, [GEORGE])
     assert r == [0, 1]
     assert len(p.user_ids) == 1, "provisioning should re-enable user"
     assert p.user_ids.active
 
-    # re-enable both user and partner to see if both get re-enabled
+    # matching deactivated partner (with a deactivated user)
     p.user_ids.active = False
     p.active = False
     r = provision_user(port, [GEORGE])
@@ -92,6 +81,33 @@ def test_upgrade_partner(env, port):
     assert p.active, "provisioning should re-enable partner"
     assert p.user_ids.active
 
+def test_duplicates(env, port):
+    """In case of duplicate data, the handler should probably not blow up, but
+    instead log a warning (so the data gets fixed eventually) and skip
+    """
+    # dupe 1: old oauth signup account & github interaction account, provisioning
+    # prioritises the github account & tries to create a user for it, which
+    # fails because the signup account has the same oauth uid (probably)
+    env['res.partner'].create({'name': 'foo', 'github_login': 'foo'})
+    env['res.users'].create({'login': 'foo@example.com', 'name': 'foo', 'email': 'foo@example.com', 'oauth_provider_id': 1, 'oauth_uid': '42'})
+    assert provision_user(port, [{
+        'name': "foo",
+        'email': 'foo@example.com',
+        'github_login': 'foo',
+        'sub': '42'
+    }]) == [0, 0]
+
+    # dupe 2: old non-oauth signup account & github interaction account, same
+    # as previous except it breaks on the login instead of the oauth_uid
+    env['res.partner'].create({'name': 'bar', 'github_login': 'bar'})
+    env['res.users'].create({'login': 'bar@example.com', 'name': 'bar', 'email': 'bar@example.com'})
+    assert provision_user(port, [{
+        'name': "bar",
+        'email': 'bar@example.com',
+        'github_login': 'bar',
+        'sub': '43'
+    }]) == [0, 0]
+
 def test_no_email(env, port):
     """ Provisioning system should ignore email-less entries
     """
@@ -182,6 +198,6 @@ def provision_user(port, users):
     })
     r.raise_for_status()
     json = r.json()
-    assert 'error' not in json
+    assert 'error' not in json, json['error']['data']['debug']
 
     return json['result']

From fc41e09f44f33ddec91ab8ef9f75a13988943cc9 Mon Sep 17 00:00:00 2001
From: Xavier-Do <xdo@odoo.com>
Date: Wed, 21 Jun 2023 10:08:14 +0200
Subject: [PATCH 014/192] [IMP] runbot: add useful indexes remaining from
 upgrade

---
 runbot/models/batch.py  | 2 +-
 runbot/models/branch.py | 4 ++--
 runbot/models/build.py  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/runbot/models/batch.py b/runbot/models/batch.py
index 6c5f4d01..eaea5f39 100644
--- a/runbot/models/batch.py
+++ b/runbot/models/batch.py
@@ -22,7 +22,7 @@ class Batch(models.Model):
     state = fields.Selection([('preparing', 'Preparing'), ('ready', 'Ready'), ('done', 'Done'), ('skipped', 'Skipped')])
     hidden = fields.Boolean('Hidden', default=False)
     age = fields.Integer(compute='_compute_age', string='Build age')
-    category_id = fields.Many2one('runbot.category', default=lambda self: self.env.ref('runbot.default_category', raise_if_not_found=False))
+    category_id = fields.Many2one('runbot.category', index=True, default=lambda self: self.env.ref('runbot.default_category', raise_if_not_found=False))
     log_ids = fields.One2many('runbot.batch.log', 'batch_id')
     has_warning = fields.Boolean("Has warning")
     base_reference_batch_id = fields.Many2one('runbot.batch')
diff --git a/runbot/models/branch.py b/runbot/models/branch.py
index daa7f64c..80079bd5 100644
--- a/runbot/models/branch.py
+++ b/runbot/models/branch.py
@@ -17,7 +17,7 @@ class Branch(models.Model):
     _sql_constraints = [('branch_repo_uniq', 'unique (name,remote_id)', 'The branch must be unique per repository !')]
 
     name = fields.Char('Name', required=True)
-    remote_id = fields.Many2one('runbot.remote', 'Remote', required=True, ondelete='cascade')
+    remote_id = fields.Many2one('runbot.remote', 'Remote', required=True, ondelete='cascade', index=True)
 
     head = fields.Many2one('runbot.commit', 'Head Commit', index=True)
     head_name = fields.Char('Head name', related='head.name', store=True)
@@ -29,7 +29,7 @@ class Branch(models.Model):
     pr_title = fields.Char('Pr Title')
     pr_body = fields.Char('Pr Body')
     pr_author = fields.Char('Pr Author')
-    
+
     pull_head_name = fields.Char(compute='_compute_branch_infos', string='PR HEAD name', readonly=1, store=True)
     pull_head_remote_id = fields.Many2one('runbot.remote', 'Pull head repository', compute='_compute_branch_infos', store=True, index=True)
     target_branch_name = fields.Char(compute='_compute_branch_infos', string='PR target branch', store=True)
diff --git a/runbot/models/build.py b/runbot/models/build.py
index 468ea9e1..0e26c4c3 100644
--- a/runbot/models/build.py
+++ b/runbot/models/build.py
@@ -51,7 +51,7 @@ class BuildParameters(models.Model):
     version_id = fields.Many2one('runbot.version', required=True, index=True)
     project_id = fields.Many2one('runbot.project', required=True, index=True)  # for access rights
     trigger_id = fields.Many2one('runbot.trigger', index=True)  # for access rights
-    create_batch_id = fields.Many2one('runbot.batch')
+    create_batch_id = fields.Many2one('runbot.batch', index=True)
     category = fields.Char('Category', index=True)  # normal vs nightly vs weekly, ...
     dockerfile_id = fields.Many2one('runbot.dockerfile', index=True, default=lambda self: self.env.ref('runbot.docker_default', raise_if_not_found=False))
     skip_requirements = fields.Boolean('Skip requirements.txt auto install')

From 72281b0c632c1d7c09a066227c2c98cbb3f31eb0 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 22 Jun 2023 14:37:49 +0200
Subject: [PATCH 015/192] [IMP] runbot_merge: allow running test suite without
 an explicit addons path

---
 conftest.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/conftest.py b/conftest.py
index 8e7b05f9..0af3c1c5 100644
--- a/conftest.py
+++ b/conftest.py
@@ -277,7 +277,7 @@ class DbDict(dict):
         with tempfile.TemporaryDirectory() as d:
             subprocess.run([
                 'odoo', '--no-http',
-                '--addons-path', self._adpath,
+                *(['--addons-path', self._adpath] if self._adpath else []),
                 '-d', db, '-i', module + ',auth_oauth',
                 '--max-cron-threads', '0',
                 '--stop-after-init',
@@ -296,7 +296,7 @@ def dbcache(request):
     dbs = DbDict(request.config.getoption('--addons-path'))
     yield dbs
     for db in dbs.values():
-        subprocess.run(['dropdb', db], check=True)
+        subprocess.run(['dropdb', '--if-exists', db], check=True)
 
 @pytest.fixture
 def db(request, module, dbcache):
@@ -369,10 +369,10 @@ def server(request, db, port, module, dummy_addons_path, tmpdir):
     if not request.config.getoption('--log-github'):
         log_handlers.append('github_requests:WARNING')
 
-    addons_path = ','.join(map(str, [
+    addons_path = ','.join(map(str, filter(None, [
         request.config.getoption('--addons-path'),
         dummy_addons_path,
-    ]))
+    ])))
 
     cov = []
     if request.config.getoption('--coverage'):

From b72fdff01d3d923b7ba37c64b8df9e13b3a42f35 Mon Sep 17 00:00:00 2001
From: Xavier-Do <xdo@odoo.com>
Date: Thu, 22 Jun 2023 15:57:19 +0200
Subject: [PATCH 016/192] [DEL] runbot: 15.0 eol


From f22ae357e343cfd09735fd35a1d097589a99d674 Mon Sep 17 00:00:00 2001
From: Xavier-Do <xdo@odoo.com>
Date: Thu, 22 Jun 2023 15:57:19 +0200
Subject: [PATCH 017/192] [DEL] runbot: 15.0 eol


From d748d4b215a72cf1b73d023f224b61ac908a4926 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 27 Jun 2023 12:51:23 +0200
Subject: [PATCH 018/192] [IMP] *: create a single template db per module to
 test

Before this, when testing in parallel (using xdist) each worker would
create its own template database (per module, so 2) then would copy
the database for each test.

This is pretty inefficient as the init of a db is quite expensive in
CPU, and when increasing the number of workers (as the test suite is
rather IO bound) this would trigger a stampede as every worker would
try to create a template at the start of the test suite, leading to
extremely high loads and degraded host performances (e.g. 16 workers
would cause a load of 20 on a 4 cores 8 thread machine, which makes
its use difficult).

Instead we can have a lockfile at a known location of the filesystem,
the first worker to need a template for a module install locks it,
creates the templates, then writes the template's name to the
lockfile.

Every other worker can then lock the lockfile and read the name out,
using the db for duplication.

Note: needs to use `os.open` because the modes of `open` apparently
can't express "open at offset 0 for reading or create for writing",
`r+` refuses to create the file, `w+` still truncates, and `a+` is
undocumented and might not allow seeking back to the start on all
systems so better avoid it.

The implementation would be simplified by using `lockfile` but that's
an additional dependency plus it's deprecated. It recommends
`fasteners` but that seems to suck (not clear if storing stuff in the
lockfile is supported, it opens the lockfile in append mode). Here the
lockfiles are sufficient to do the entire thing.

Conveniently, this turns out to improve *both* walltime CPU time
compared to the original version, likely because while workers now
have to wait on whoever is creating the template they're not competing
for resources with it.
---
 conftest.py | 46 +++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 39 insertions(+), 7 deletions(-)

diff --git a/conftest.py b/conftest.py
index 0af3c1c5..329b3524 100644
--- a/conftest.py
+++ b/conftest.py
@@ -46,6 +46,7 @@ import collections
 import configparser
 import contextlib
 import copy
+import fcntl
 import functools
 import http.client
 import itertools
@@ -88,11 +89,20 @@ def pytest_addoption(parser):
              "blow through the former); localtunnel has no rate-limiting but "
              "the servers are way less reliable")
 
+def is_manager(config):
+    return not hasattr(config, 'workerinput')
 
 # noinspection PyUnusedLocal
 def pytest_configure(config):
     sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'mergebot_test_utils'))
 
+def pytest_unconfigure(config):
+    if not is_manager(config):
+        return
+
+    for c in config._tmp_path_factory.getbasetemp().iterdir():
+        if c.is_file() and c.name.startswith('template-'):
+            subprocess.run(['dropdb', '--if-exists', c.read_text(encoding='utf-8')])
 
 @pytest.fixture(scope='session', autouse=True)
 def _set_socket_timeout():
@@ -269,12 +279,26 @@ def tunnel(pytestconfig, port):
         raise ValueError("Unsupported %s tunnel method" % tunnel)
 
 class DbDict(dict):
-    def __init__(self, adpath):
+    def __init__(self, adpath, shared_dir):
         super().__init__()
         self._adpath = adpath
+        self._shared_dir = shared_dir
     def __missing__(self, module):
-        self[module] = db = 'template_%s' % uuid.uuid4()
-        with tempfile.TemporaryDirectory() as d:
+        with contextlib.ExitStack() as atexit:
+            f = atexit.enter_context(os.fdopen(os.open(
+                self._shared_dir / f'template-{module}',
+                os.O_CREAT | os.O_RDWR
+            ), mode="r+", encoding='utf-8'))
+            fcntl.lockf(f, fcntl.LOCK_EX)
+            atexit.callback(fcntl.lockf, f, fcntl.LOCK_UN)
+
+            db = f.read()
+            if db:
+                self[module] = db
+                return db
+
+            d = atexit.enter_context(tempfile.TemporaryDirectory())
+            self[module] = db = 'template_%s' % uuid.uuid4()
             subprocess.run([
                 'odoo', '--no-http',
                 *(['--addons-path', self._adpath] if self._adpath else []),
@@ -286,17 +310,25 @@ class DbDict(dict):
                 check=True,
                 env={**os.environ, 'XDG_DATA_HOME': d}
             )
+            f.write(db)
+            f.flush()
+            os.fsync(f.fileno())
+
         return db
 
 @pytest.fixture(scope='session')
-def dbcache(request):
+def dbcache(request, tmp_path_factory):
     """ Creates template DB once per run, then just duplicates it before
     starting odoo and running the testcase
     """
-    dbs = DbDict(request.config.getoption('--addons-path'))
+    shared_dir = tmp_path_factory.getbasetemp()
+    if not is_manager(request.config):
+        # xdist workers get a subdir as their basetemp, so we need to go one
+        # level up to deref it
+        shared_dir = shared_dir.parent
+
+    dbs = DbDict(request.config.getoption('--addons-path'), shared_dir)
     yield dbs
-    for db in dbs.values():
-        subprocess.run(['dropdb', '--if-exists', db], check=True)
 
 @pytest.fixture
 def db(request, module, dbcache):

From aefbdaf974777a1462d234cd2842097ab0fc61c0 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 4 Jul 2023 13:08:40 +0200
Subject: [PATCH 019/192] [IMP] *: client side sorted implementation

Allow sorting by a callback. Sort remains client-side.
---
 conftest.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/conftest.py b/conftest.py
index 329b3524..a244f2af 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1185,9 +1185,13 @@ class Model:
 
     # because sorted is not xmlrpc-compatible (it doesn't downgrade properly)
     def sorted(self, field):
-        rs = self.read([field])
-        rs.sort(key=lambda r: r[field])
-        return Model(self._env, self._model, [r['id'] for r in rs])
+        fn = field if callable(field) else lambda r: r[field]
+
+        return Model(self._env, self._model, (
+            id
+            for record in sorted(self, key=fn)
+            for id in record.ids
+        ))
 
     def __getitem__(self, index):
         if isinstance(index, str):

From 81ce4ea02b76af4dcc859739985a41ff8c073b6e Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 5 Jul 2023 15:11:40 +0200
Subject: [PATCH 020/192] [IMP] rewrite /forwardport/outstanding

- add support for authorship (not just approval)
- make display counts directly
- fix `state` filter: postgres can't do negative index lookups
- add indexes for author and reviewed_by as we look them up
- ensure we handle the entire source filtering via a single subquery

Closes #778
---
 forwardport/controllers.py                 |  79 +++++++++++
 forwardport/data/views.xml                 | 152 +++++++++++++--------
 forwardport/models/project.py              |  25 ----
 runbot_merge/models/pull_requests.py       |   4 +-
 runbot_merge/static/scss/runbot_merge.scss |  11 ++
 5 files changed, 185 insertions(+), 86 deletions(-)

diff --git a/forwardport/controllers.py b/forwardport/controllers.py
index af464dcb..620eab96 100644
--- a/forwardport/controllers.py
+++ b/forwardport/controllers.py
@@ -1,7 +1,14 @@
+import collections
+import datetime
 import pathlib
 
+import werkzeug.urls
+
+from odoo.http import route, request
+from odoo.osv import expression
 from odoo.addons.runbot_merge.controllers.dashboard import MergebotDashboard
 
+DEFAULT_DELTA = datetime.timedelta(days=7)
 class Dashboard(MergebotDashboard):
     def _entries(self):
         changelog = pathlib.Path(__file__).parent / 'changelog'
@@ -13,3 +20,75 @@ class Dashboard(MergebotDashboard):
             for d in changelog.iterdir()
         ]
 
+
+    @route('/forwardport/outstanding', type='http', methods=['GET'], auth="user", website=True, sitemap=False)
+    def outstanding(self, partner=0, authors=True, reviewers=True, group=0):
+        Partners = request.env['res.partner']
+        PullRequests = request.env['runbot_merge.pull_requests']
+        partner = Partners.browse(int(partner))
+        group = Partners.browse(int(group))
+        authors = int(authors)
+        reviewers = int(reviewers)
+        link = lambda **kw: '?' + werkzeug.urls.url_encode({'partner': partner.id or 0, 'authors': authors, 'reviewers': reviewers, **kw, })
+        groups = Partners.search([('is_company', '=', True), ('child_ids', '!=', False)])
+        if not (authors or reviewers):
+            return request.render('forwardport.outstanding', {
+                'authors': 0,
+                'reviewers': 0,
+                'single': partner,
+                'culprits': partner,
+                'groups': groups,
+                'current_group': group,
+                'outstanding': [],
+                'outstanding_per_author': {partner: 0},
+                'outstanding_per_reviewer': {partner: 0},
+                'link': link,
+            })
+
+        source_filter = [('merge_date', '<', datetime.datetime.now() - DEFAULT_DELTA)]
+        partner_filter = []
+        if partner or group:
+            if partner:
+                suffix = ''
+                arg = partner.id
+            else:
+                suffix = '.commercial_partner_id'
+                arg = group.id
+
+            if authors:
+                partner_filter.append([(f'author{suffix}', '=', arg)])
+            if reviewers:
+                partner_filter.append([(f'reviewed_by{suffix}', '=', arg)])
+
+            source_filter.extend(expression.OR(partner_filter))
+
+        outstanding = PullRequests.search([
+            ('state', 'in', ['opened', 'validated', 'approved', 'ready', 'error']),
+            ('source_id', 'in', PullRequests._search(source_filter)),
+        ])
+        outstanding_per_author = collections.Counter()
+        outstanding_per_reviewer = collections.Counter()
+        outstandings = []
+        for source in outstanding.mapped('source_id').sorted('merge_date'):
+            outstandings.append({
+                'source': source,
+                'prs': source.forwardport_ids.filtered(lambda p: p.state not in ['merged', 'closed']),
+            })
+            if authors:
+                outstanding_per_author[source.author] += 1
+            if reviewers and source:
+                outstanding_per_reviewer[source.reviewed_by] += 1
+
+        culprits = Partners.browse(p.id for p, _ in (outstanding_per_reviewer + outstanding_per_author).most_common())
+        return request.render('forwardport.outstanding', {
+            'authors': authors,
+            'reviewers': reviewers,
+            'single': partner,
+            'culprits': culprits,
+            'groups': groups,
+            'current_group': group,
+            'outstanding_per_author': outstanding_per_author,
+            'outstanding_per_reviewer': outstanding_per_reviewer,
+            'outstanding': outstandings,
+            'link': link,
+        })
diff --git a/forwardport/data/views.xml b/forwardport/data/views.xml
index c25bb311..1b6069c2 100644
--- a/forwardport/data/views.xml
+++ b/forwardport/data/views.xml
@@ -30,67 +30,101 @@
         <t t-else="">bg-warning</t>
     </template>
 
-    <record id="forwardport.outstanding_fp" model="website.page">
-        <field name="name">Outstanding forward ports</field>
-        <field name="type">qweb</field>
-        <field name="url">/forwardport/outstanding</field>
-        <field name="website_indexed" eval="False"/>
-        <field name="is_published">True</field>
-        <field name="key">forwardport.outstanding_fp</field>
-        <field name="arch" type="xml">
-            <t name="Outstanding forward ports" t-name="forwardport.outstanding_fp">
-                <t t-call="website.layout">
-                    <t t-set="hof" t-value="env['runbot_merge.pull_requests']._hall_of_shame()"/>
-                    <div id="wrap" class="oe_structure oe_empty"><div class="container-fluid">
-                        <ul class="alert bg-light list-inline">
-                            <span t-foreach="hof.reviewers" t-as="count" class="list-inline-item">
-                                <a t-attf-href="?reviewer={{count[0].id}}"
-                                   t-field="count[0].display_name"
-                                   t-att-title="count[1]"
-                                />
-                            </span>
-                        </ul>
-                        <h1>List of pull requests with outstanding forward ports</h1>
-                        <t t-set="reviewer" t-value="env['res.partner'].browse(int(request.params.get('reviewer') or 0))"/>
-                        <form method="get" action="" id="reset-filter"/>
-                        <h2 t-if="reviewer" class="text-muted">
-                            merged by <span t-field="reviewer.display_name" t-attf-title="@{{reviewer.github_login}}"/>
-                            <button form="reset-filter" type="submit"
-                                    name="reviewer" value=""
-                                    title="See All" class="btn fa fa-times"/>
-                        </h2>
-                        <dl><t t-foreach="hof.outstanding" t-as="x">
-                            <t t-set="source" t-value="x[0]"/>
-                            <t t-if="not reviewer or source.reviewed_by == reviewer">
-                                <dt>
-                                    <a t-att-href="source.url"><span t-field="source.display_name"/></a>
-                                    by <span t-field="source.author.display_name"
-                                             t-attf-title="@{{source.author.github_login}}"/>
-                                    merged <span t-field="source.merge_date"
-                                                 t-options="{'widget': 'relative'}"
-                                                 t-att-title="source.merge_date"/>
-                                    <t t-if="not reviewer">
-                                        by <span t-field="source.reviewed_by.display_name"
-                                                 t-attf-title="@{{source.reviewed_by.github_login}}"/>
-                                    </t>
-                                </dt>
-                                <dd>
-                                    Outstanding forward-ports:
-                                    <ul>
-                                        <li t-foreach="x.prs" t-as="p">
-                                            <a t-att-href="p.url"><span t-field="p.display_name"/></a>
-                                            (<span t-field="p.state"/>)
-                                            targeting <span t-field="p.target.name"/>
-                                        </li>
-                                    </ul>
-                                </dd>
+    <template id="outstanding" name="Outstanding forward ports">
+        <t t-call="website.layout">
+            <div id="wrap" class="oe_structure oe_empty"><div class="container-fluid">
+                <div class="alert bg-light outstanding-partners">
+                    <t t-foreach="groups" t-as="group">
+                        <span>
+                            <t t-if="group == current_group">
+                                <span class="bg-primary" t-out="group.display_name"/>
+                                <a t-att-href="link()" class="btn fa fa-times p-0"/>
                             </t>
-                        </t></dl>
-                    </div></div>
+                            <t t-else="">
+                                <a t-att-href="link(group=group.id, partner=0)" t-out="group.display_name"/>
+                            </t>
+                        </span>
+                    </t>
+                </div>
+                <div class="alert bg-light outstanding-partners">
+                    <t t-foreach="culprits" t-as="culprit">
+                        <t t-set="approved" t-value="outstanding_per_reviewer[culprit]"/>
+                        <t t-set="created" t-value="outstanding_per_author[culprit]"/>
+                        <a t-att-href="link(partner=culprit.id)"
+                           t-attf-title="approved {{approved}}, created {{created}}"
+                        ><t t-out="culprit.display_name"/>:
+                            <t t-if="approved" t-out="approved"/>
+                            <t t-if="approved and created"> + </t>
+                            <t t-if="created" t-out="created"/>
+                        </a>
+                    </t>
+                </div>
+                <t t-if="not single">
+                    by
+                    <span class="btn-group btn-group-toggle">
+                        <a t-att-href="link(authors=1, reviewers=1)"
+                           t-attf-class="btn btn-sm btn-secondary {{'active' if authors and reviewers else ''}}">
+                            both
+                        </a>
+                        <a t-att-href="link(authors=1, reviewers=0)"
+                           t-attf-class="btn btn-sm btn-secondary {{'active' if authors and not reviewers else ''}}">
+                            creators
+                        </a>
+                        <a t-att-href="link(reviewers=1, authors=0)"
+                           t-attf-class="btn btn-sm btn-secondary {{'active' if reviewers and not authors else ''}}">
+                            reviewers
+                        </a>
+                    </span>
                 </t>
-            </t>
-        </field>
-    </record>
+                <h1>List of pull requests with outstanding forward ports</h1>
+                <h2 t-if="single">
+                    for <span t-field="single.display_name" t-attf-title="@{{single.github_login}}"/>
+                    <a t-att-href="link(partner=0)" title="All Users" class="btn fa fa-times"/>
+
+                    <span class="btn-group btn-group-toggle">
+                        <a t-att-href="link(authors=1, reviewers=1)"
+                           t-attf-class="btn btn-sm btn-secondary {{'active' if authors and reviewers else ''}}">
+                            both
+                        </a>
+                        <a t-att-href="link(authors=1, reviewers=0)"
+                           t-attf-class="btn btn-sm btn-secondary {{'active' if authors and not reviewers else ''}}">
+                            created
+                        </a>
+                        <a t-att-href="link(reviewers=1, authors=0)"
+                           t-attf-class="btn btn-sm btn-secondary {{'active' if reviewers and not authors else ''}}">
+                            reviewed
+                        </a>
+                    </span>
+                </h2>
+
+                <dl><t t-foreach="outstanding" t-as="x">
+                    <t t-set="source" t-value="x['source']"/>
+                    <t t-if="not single or source.reviewed_by == single or source.author == single">
+                        <dt>
+                            <a t-att-href="source.url"><span t-field="source.display_name"/></a>
+                            created by <span t-field="source.author.display_name"
+                                             t-attf-title="@{{source.author.github_login}}"/>
+                            merged <span t-field="source.merge_date"
+                                         t-options="{'widget': 'relative'}"
+                                         t-att-title="source.merge_date"/>
+                            by <span t-field="source.reviewed_by.display_name"
+                                     t-attf-title="@{{source.reviewed_by.github_login}}"/>
+                        </dt>
+                        <dd>
+                            Outstanding forward-ports:
+                            <ul>
+                                <li t-foreach="x['prs']" t-as="p">
+                                    <a t-att-href="p.url"><span t-field="p.display_name"/></a>
+                                    (<span t-field="p.state"/>)
+                                    targeting <span t-field="p.target.name"/>
+                                </li>
+                            </ul>
+                        </dd>
+                    </t>
+                </t></dl>
+            </div></div>
+        </t>
+    </template>
 
     <template id="view_pull_request" inherit_id="runbot_merge.view_pull_request">
         <xpath expr="//dl[hasclass('runbot-merge-fields')]" position="inside">
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 720000e7..51974020 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -1114,31 +1114,6 @@ stderr:
             ('source_id.merge_date', '<', cutoff),
         ], order='source_id, id'), lambda p: p.source_id)
 
-    def _hall_of_shame(self):
-        """Provides data for the HOS view
-
-        * outstanding forward ports per reviewer
-        * pull requests with outstanding forward ports, oldest-merged first
-        """
-        cutoff_dt = datetime.datetime.now() - DEFAULT_DELTA
-        outstanding = self.env['runbot_merge.pull_requests'].search([
-            ('source_id', '!=', False),
-            ('state', 'not in', ['merged', 'closed']),
-            ('source_id.merge_date', '<', cutoff_dt),
-        ], order=None)
-        # only keep merged because apparently some PRs are in a weird spot
-        # where they're sources but closed?
-        sources = outstanding.mapped('source_id').filtered('merge_date').sorted('merge_date')
-        outstandings = []
-        reviewers = collections.Counter()
-        for source in sources:
-            outstandings.append(Outstanding(source=source, prs=source.forwardport_ids & outstanding))
-            reviewers[source.reviewed_by] += 1
-        return HallOfShame(
-            reviewers=reviewers.most_common(),
-            outstanding=outstandings,
-        )
-
     def _reminder(self):
         cutoff = self.env.context.get('forwardport_updated_before') \
               or fields.Datetime.to_string(datetime.datetime.now() - DEFAULT_DELTA)
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 19c63aef..d48ad505 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -527,7 +527,7 @@ class PullRequests(models.Model):
     ], default='opened', index=True)
 
     number = fields.Integer(required=True, index=True, group_operator=None)
-    author = fields.Many2one('res.partner')
+    author = fields.Many2one('res.partner', index=True)
     head = fields.Char(required=True)
     label = fields.Char(
         required=True, index=True,
@@ -545,7 +545,7 @@ class PullRequests(models.Model):
     ], default=False)
     method_warned = fields.Boolean(default=False)
 
-    reviewed_by = fields.Many2one('res.partner')
+    reviewed_by = fields.Many2one('res.partner', index=True)
     delegates = fields.Many2many('res.partner', help="Delegate reviewers, not intrinsically reviewers but can review this PR")
     priority = fields.Integer(default=2, index=True, group_operator=None)
 
diff --git a/runbot_merge/static/scss/runbot_merge.scss b/runbot_merge/static/scss/runbot_merge.scss
index 2b98cf32..2e6577b3 100644
--- a/runbot_merge/static/scss/runbot_merge.scss
+++ b/runbot_merge/static/scss/runbot_merge.scss
@@ -110,3 +110,14 @@ dl.runbot-merge-fields {
 .staging-statuses {
   cursor: wait;
 }
+
+/* forwardport */
+.outstanding-partners > * {
+  @extend .pt-1;
+  // because there's a trailing space which is annoying to remove, which plays
+  // the role of padding-right
+  @extend .pl-1;
+  @extend .text-nowrap;
+  // works better for the left edge of the *box*
+  @extend .border-left;
+}

From 5bce73c97dc4a1b6e81165d27e9377ae32a3a073 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 6 Jul 2023 10:28:52 +0200
Subject: [PATCH 021/192] [IMP] *: optimise loading of home page

Fix outstanding query to make a positive `state` filtering, instead of
negative, matching 3b52b1aace8674259812a76b1566260937dbcacb.

Also manually create a map of stagings (grouped by branch) sharing a
single prefetch set.

For odoo the mergebot home page has 12 branches in the odoo project
and 8 in spreadsheet, 6 stagings each. This means 120 queries to
retrieve all the heads (Odoo stagings have 5 heads and spreadsheet
have 1, but that seems immaterial).

By fixing `_compute_statuses` and creating a single prefetch set for
all stagings of all branches we can fetch all the commits in a single
query instead of 120.
---
 forwardport/data/views.xml            |  3 +--
 runbot_merge/controllers/dashboard.py | 19 ++++++++++++++++++-
 runbot_merge/models/pull_requests.py  | 23 ++++++++++++++++++-----
 runbot_merge/views/templates.xml      |  4 ++--
 4 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/forwardport/data/views.xml b/forwardport/data/views.xml
index 1b6069c2..08ef8177 100644
--- a/forwardport/data/views.xml
+++ b/forwardport/data/views.xml
@@ -10,8 +10,7 @@
         <xpath expr="//div[@id='alerts']" position="inside">
             <t t-if="env['runbot_merge.pull_requests'].check_access_rights('read', False)">
                 <t t-set="outstanding" t-value="env['runbot_merge.pull_requests'].search_count([
-                    ('source_id', '!=', False),
-                    ('state', 'not in', ['merged', 'closed']),
+                    ('state', 'in', ['opened', 'validated', 'approved', 'ready', 'error']),
                     ('source_id.merge_date', '&lt;', datetime.datetime.now() - relativedelta(days=3)),
                 ])"/>
                 <div t-if="outstanding != 0" class="alert col-md-12 alert-warning mb-0">
diff --git a/runbot_merge/controllers/dashboard.py b/runbot_merge/controllers/dashboard.py
index f15c52d2..8a173c77 100644
--- a/runbot_merge/controllers/dashboard.py
+++ b/runbot_merge/controllers/dashboard.py
@@ -13,8 +13,25 @@ LIMIT = 20
 class MergebotDashboard(Controller):
     @route('/runbot_merge', auth="public", type="http", website=True, sitemap=True)
     def dashboard(self):
+        projects = request.env['runbot_merge.project'].with_context(active_test=False).sudo().search([])
+        stagings = {
+            branch: projects.env['runbot_merge.stagings'].search([
+                ('target', '=', branch.id)], order='staged_at desc', limit=6)
+            for project in projects
+            for branch in project.branch_ids
+            if branch.active
+        }
+        prefetch_set = list({
+            id
+            for stagings in stagings.values()
+            for id in stagings.ids
+        })
+        for st in stagings.values():
+            st._prefetch_ids = prefetch_set
+
         return request.render('runbot_merge.dashboard', {
-            'projects': request.env['runbot_merge.project'].with_context(active_test=False).sudo().search([]),
+            'projects': projects,
+            'stagings_map': stagings,
         })
 
     @route('/runbot_merge/<int:branch_id>', auth='public', type='http', website=True, sitemap=False)
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index d48ad505..5b3c23d5 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1813,12 +1813,25 @@ class Stagings(models.Model):
         (repo, context, state, url)
         """
         Commits = self.env['runbot_merge.commit']
+        heads = {
+            head: repo
+            for st in self
+            for repo, head in json.loads(st.heads).items()
+            if not repo.endswith('^')
+        }
+        all_heads = Commits.search([('sha', 'in', list(heads))])
+        commits_map = {commit.sha: commit.id for commit in all_heads}
+
         for st in self:
-            heads = {
-                head: repo for repo, head in json.loads(st.heads).items()
-                if not repo.endswith('^')
-            }
-            commits = st.head_ids = Commits.search([('sha', 'in', list(heads.keys()))])
+            commits = st.head_ids = Commits._browse(
+                self.env,
+                tuple(
+                    commits_map[h]
+                    for repo, h in json.loads(st.heads).items()
+                    if not repo.endswith('^')
+                ),
+                all_heads._prefetch_ids
+            )
             if st.statuses_cache:
                 st.statuses = json.loads(st.statuses_cache)
                 continue
diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index 73e8e883..34b403a7 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -72,7 +72,7 @@
         <t t-call="website.layout">
             <div id="wrap"><div class="container-fluid">
                 <t t-call="runbot_merge.alerts"/>
-                <section t-foreach="projects.with_context(active_test=False)" t-as="project" class="row">
+                <section t-foreach="projects" t-as="project" class="row">
                     <h1 class="col-md-12"><t t-esc="project.name"/></h1>
                     <div class="col-md-12">
                         key:
@@ -154,7 +154,7 @@
     <template id="stagings" name="mergebot branch stagings">
         <t t-set="repo_statuses" t-value="branch.project_id.repo_ids.having_branch(branch).status_ids"/>
         <ul class="list-unstyled stagings">
-            <t t-foreach="branch.env['runbot_merge.stagings'].search([('target', '=', branch.id)], order='staged_at desc', limit=6)" t-as="staging">
+            <t t-foreach="stagings_map[branch]" t-as="staging">
                 <t t-set="success" t-value="staging.state == 'success'"/>
                 <t t-set="failure" t-value="staging.state == 'failure'"/>
                 <t t-set="pending" t-value="staging.active and (not staging.state or staging.state == 'pending')"/>

From 780e20bfd6e56e77b7002d701192318aa5beb5e2 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 6 Jul 2023 14:12:38 +0200
Subject: [PATCH 022/192] [IMP] runbot_merge: filtering options and UX on
 stagings list

Allow filtering stagings by state (success or failure), and provide a
control to explicitly update the staging date limit.

Should make it easier to drill through stagings when looking for
specific information.

Related to #751
---
 runbot_merge/controllers/dashboard.py | 15 ++++++++++-----
 runbot_merge/models/pull_requests.py  |  2 +-
 runbot_merge/views/templates.xml      | 14 +++++++++++++-
 3 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/runbot_merge/controllers/dashboard.py b/runbot_merge/controllers/dashboard.py
index 8a173c77..b06ee8bf 100644
--- a/runbot_merge/controllers/dashboard.py
+++ b/runbot_merge/controllers/dashboard.py
@@ -35,19 +35,24 @@ class MergebotDashboard(Controller):
         })
 
     @route('/runbot_merge/<int:branch_id>', auth='public', type='http', website=True, sitemap=False)
-    def stagings(self, branch_id, until=None):
+    def stagings(self, branch_id, until=None, state=''):
         branch = request.env['runbot_merge.branch'].browse(branch_id).sudo().exists()
         if not branch:
             raise werkzeug.exceptions.NotFound()
 
-        stagings = request.env['runbot_merge.stagings'].with_context(active_test=False).sudo().search([
-            ('target', '=', branch.id),
-            ('staged_at', '<=', until) if until else (True, '=', True),
-        ], order='staged_at desc', limit=LIMIT+1)
+        staging_domain = [('target', '=', branch.id)]
+        if until:
+            staging_domain.append(('staged_at', '<=', until))
+        if state:
+            staging_domain.append(('state', '=', state))
+
+        stagings = request.env['runbot_merge.stagings'].with_context(active_test=False).sudo().search(staging_domain, order='staged_at desc', limit=LIMIT + 1)
 
         return request.render('runbot_merge.branch_stagings', {
             'branch': branch,
             'stagings': stagings[:LIMIT],
+            'until': until,
+            'state': state,
             'next': stagings[-1].staged_at if len(stagings) > LIMIT else None,
         })
 
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 5b3c23d5..d9b63411 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1764,7 +1764,7 @@ class Stagings(models.Model):
         ('pending', 'Pending'),
         ('cancelled', "Cancelled"),
         ('ff_failed', "Fast forward failed")
-    ], default='pending')
+    ], default='pending', index=True)
     active = fields.Boolean(default=True)
 
     staged_at = fields.Datetime(default=fields.Datetime.now, index=True)
diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index 34b403a7..1a5fd4a1 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -199,6 +199,18 @@
                 <section class="row">
                     <h1 class="col-md-12"><t t-esc="branch.project_id.name"/>: <t t-esc="branch.name"/></h1>
                 </section>
+                <form method="get">
+                    <label for="until">Staged before:</label>
+                    <input type="datetime-local" name="until" t-att-value="until"/>
+                    (UTC)
+                    <label for="state">State:</label>
+                    <select name="state">
+                        <option t-att-selected="'selected' if not state else None"/>
+                        <option t-att-selected="'selected' if state == 'success' else None" value="success">Success</option>
+                        <option t-att-selected="'selected' if state == 'failure' else None" value="failure">Failure</option>
+                    </select>
+                    <button type="submit">Apply</button>
+                </form>
                 <table>
                     <t t-foreach="stagings" t-as="staging">
                         <t t-set="success"
@@ -270,7 +282,7 @@
                     </t>
                 </table>
                 <t t-if="next">
-                    <a t-attf-href="/runbot_merge/{{branch.id}}?until={{next}}">
+                    <a t-attf-href="/runbot_merge/{{branch.id}}?until={{next}}&amp;state={{state}}">
                         Next >
                     </a>
                 </t>

From a692163f6eb46c91f34384f5f6513488c9554647 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 6 Jul 2023 14:23:09 +0200
Subject: [PATCH 023/192] [IMP] runbot_merge: add quick jump from stagings to
 PRs

In the backend, the intermediate jump through batches is really not
convenient (even if we kinda have to jump through batches *anyway*).

Fixes #751
---
 runbot_merge/models/pull_requests.py |  6 ++++++
 runbot_merge/views/mergebot.xml      | 16 ++++++++++++----
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index d9b63411..18294ea9 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1758,6 +1758,7 @@ class Stagings(models.Model):
         'runbot_merge.batch', 'staging_id',
         context={'active_test': False},
     )
+    pr_ids = fields.One2many('runbot_merge.pull_requests', compute='_compute_prs')
     state = fields.Selection([
         ('success', 'Success'),
         ('failure', 'Failure'),
@@ -1859,6 +1860,11 @@ class Stagings(models.Model):
                 + datetime.timedelta(minutes=st.target.project_id.ci_timeout)
             )
 
+    @api.depends('batch_ids.prs')
+    def _compute_prs(self):
+        for staging in self:
+            staging.pr_ids = staging.batch_ids.prs
+
     def _validate(self):
         Commits = self.env['runbot_merge.commit']
         for s in self:
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index 9cc3168b..ab5c8ee4 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -197,11 +197,19 @@
                             </tree>
                         </field>
                     </group>
-                    <group string="Batches">
-                        <field name="batch_ids" colspan="4" nolabel="1">
+                    <group string="PRs">
+                        <field name="pr_ids" colspan="4" nolabel="1" readonly="1">
                             <tree>
-                                <field name="prs" widget="many2many_tags"
-                                       options="{'no_quick_create': True}"/>
+                                <field name="display_name"/>
+                                <field name="github_url" widget="url"/>
+                                <field name="url" widget="url"/>
+                            </tree>
+                        </field>
+                    </group>
+                    <group string="Batches">
+                        <field name="batch_ids" colspan="4" nolabel="1" readonly="1">
+                            <tree>
+                                <field name="prs" widget="many2many_tags"/>
                             </tree>
                         </field>
                     </group>

From cdffa83191f6e810f18965dda104eb0bc83e65ee Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 10 Aug 2023 13:21:21 +0200
Subject: [PATCH 024/192] [IMP] runbot_merge, forwardport: minor cleanups

Remove unused imports, unnecessary f-strings, dead code, fix
less-than-ideal operators.
---
 forwardport/models/project.py                  |  3 +--
 forwardport/tests/test_limit.py                |  4 ----
 runbot_merge/github.py                         |  2 +-
 .../migrations/13.0.1.6/pre-migration.py       |  3 ---
 runbot_merge/models/pull_requests.py           |  1 -
 runbot_merge/sentry.py                         |  2 +-
 runbot_merge/tests/test_basic.py               | 18 +++++++++---------
 runbot_merge/tests/test_disabled_branch.py     |  2 +-
 runbot_merge/tests/test_oddities.py            |  2 +-
 runbot_merge/tests/test_provisioning.py        |  4 ++--
 10 files changed, 16 insertions(+), 25 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 51974020..cf60f68a 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -13,7 +13,6 @@ it up), ...
 """
 import ast
 import base64
-import collections
 import contextlib
 import datetime
 import itertools
@@ -421,7 +420,7 @@ class PullRequests(models.Model):
                 limit = next(tokens, None)
                 ping = True
                 if not self._pr_acl(author).is_author:
-                    msg = "you can't set a forward-port limit.".format(login)
+                    msg = "you can't set a forward-port limit."
                 elif not limit:
                     msg = "please provide a branch to forward-port to."
                 else:
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index db13f535..ecdfd585 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -1,8 +1,4 @@
-# -*- coding: utf-8 -*-
 import collections
-import time
-
-import pytest
 
 from utils import seen, Commit, make_basic
 
diff --git a/runbot_merge/github.py b/runbot_merge/github.py
index e5e5fa9e..5b11033b 100644
--- a/runbot_merge/github.py
+++ b/runbot_merge/github.py
@@ -71,7 +71,7 @@ class GH(object):
         """
         req = response.request
         url = werkzeug.urls.url_parse(req.url)
-        if url.netloc is not 'api.github.com':
+        if url.netloc != 'api.github.com':
             return
 
         body = '' if not req.body else ('\n' + pprint.pformat(json.loads(req.body.decode()), indent=4))
diff --git a/runbot_merge/migrations/13.0.1.6/pre-migration.py b/runbot_merge/migrations/13.0.1.6/pre-migration.py
index a7903018..639c5013 100644
--- a/runbot_merge/migrations/13.0.1.6/pre-migration.py
+++ b/runbot_merge/migrations/13.0.1.6/pre-migration.py
@@ -1,6 +1,3 @@
-import collections
-
-
 def migrate(cr, version):
     """ Status overrides: o2m -> m2m
     """
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 18294ea9..2873dbcf 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1682,7 +1682,6 @@ class FeedbackTemplate(models.Model):
         except Exception:
             _logger.exception("Failed to render template %s", self.get_external_id())
             raise
-            return None
 
 class Commit(models.Model):
     """Represents a commit onto which statuses might be posted,
diff --git a/runbot_merge/sentry.py b/runbot_merge/sentry.py
index 9c249e9e..48bdd48e 100644
--- a/runbot_merge/sentry.py
+++ b/runbot_merge/sentry.py
@@ -9,7 +9,7 @@ from odoo import http
 from odoo.addons.base.models.ir_cron import ir_cron
 from odoo.http import WebRequest
 
-from .exceptions import FastForwardError, Mismatch, MergeError, Unmergeable
+from .exceptions import FastForwardError, MergeError, Unmergeable
 
 
 def delegate(self, attr):
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 305a79e2..ccc8e36f 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -7,7 +7,7 @@ from unittest import mock
 
 import pytest
 import requests
-from lxml import html, etree
+from lxml import html
 
 import odoo
 from utils import _simple_init, seen, re_matches, get_partner, Commit, pr_page, to_pr, part_of
@@ -1776,9 +1776,9 @@ commits, I need to know how to merge it:
         with repo:
             root = repo.make_commits(None, Commit("root", tree={'a': 'a'}), ref='heads/master')
 
-            repo.make_commits(root, Commit('C', tree={'a': 'b'}), ref=f'heads/change')
+            repo.make_commits(root, Commit('C', tree={'a': 'b'}), ref='heads/change')
             pr = repo.make_pr(title="title", body=f'first\n{separator}\nsecond',
-                              target='master', head=f'change')
+                              target='master', head='change')
             repo.post_status(pr.head, 'success', 'legal/cla')
             repo.post_status(pr.head, 'success', 'ci/runbot')
             pr.post_comment('hansen r+ merge', config['role_reviewer']['token'])
@@ -1806,7 +1806,7 @@ commits, I need to know how to merge it:
         with repo:
             root = repo.make_commits(None, Commit("root", tree={'a': 'a'}), ref='heads/master')
 
-            repo.make_commits(root, Commit('C', tree={'a': 'b'}), ref=f'heads/change')
+            repo.make_commits(root, Commit('C', tree={'a': 'b'}), ref='heads/change')
             pr = repo.make_pr(title="title", body="""\
 Title
 ---
@@ -1818,7 +1818,7 @@ This is more text
 ***
 removed
 """,
-                              target='master', head=f'change')
+                              target='master', head='change')
             repo.post_status(pr.head, 'success', 'legal/cla')
             repo.post_status(pr.head, 'success', 'ci/runbot')
             pr.post_comment('hansen r+ merge', config['role_reviewer']['token'])
@@ -1853,8 +1853,8 @@ removed
         with repo:
             root = repo.make_commits(None, Commit("root", tree={'a': 'a'}), ref='heads/master')
 
-            repo.make_commits(root, Commit('Commit\n\nfirst\n***\nsecond', tree={'a': 'b'}), ref=f'heads/change')
-            pr = repo.make_pr(title="PR", body=f'first\n***\nsecond',
+            repo.make_commits(root, Commit('Commit\n\nfirst\n***\nsecond', tree={'a': 'b'}), ref='heads/change')
+            pr = repo.make_pr(title="PR", body='first\n***\nsecond',
                               target='master', head='change')
             repo.post_status(pr.head, 'success', 'legal/cla')
             repo.post_status(pr.head, 'success', 'ci/runbot')
@@ -2007,7 +2007,7 @@ Part-of: {pr_id.display_name}"""
         assert log_to_node(repo.log('heads/master')), expected
 
     def test_squash_merge(self, repo, env, config, users):
-        other_user = requests.get(f'https://api.github.com/user', headers={
+        other_user = requests.get('https://api.github.com/user', headers={
             'Authorization': 'token %s' % config['role_other']['token'],
         }).json()
         other_user = {
@@ -2089,7 +2089,7 @@ Signed-off-by: {get_partner(env, users["reviewer"]).formatted_email}\
 
         # FIXME: should probably get the token from the project to be sure it's
         #        the bot user
-        current_user = repo._session.get(f'https://api.github.com/user').json()
+        current_user = repo._session.get('https://api.github.com/user').json()
         current_user = {
             'name': current_user['name'] or current_user['login'],
             # FIXME: not guaranteed
diff --git a/runbot_merge/tests/test_disabled_branch.py b/runbot_merge/tests/test_disabled_branch.py
index 2cff73ce..17282985 100644
--- a/runbot_merge/tests/test_disabled_branch.py
+++ b/runbot_merge/tests/test_disabled_branch.py
@@ -50,7 +50,7 @@ def test_existing_pr_disabled_branch(env, project, make_repo, setreviewers, conf
     assert not branch_id.active_staging_id
     assert staging_id.state == 'cancelled', \
         "closing the PRs should have canceled the staging"
-    assert staging_id.reason == f"Target branch deactivated by 'admin'."
+    assert staging_id.reason == "Target branch deactivated by 'admin'."
 
     p = pr_page(page, pr)
     target = dict(zip(
diff --git a/runbot_merge/tests/test_oddities.py b/runbot_merge/tests/test_oddities.py
index f0a501ce..d8121629 100644
--- a/runbot_merge/tests/test_oddities.py
+++ b/runbot_merge/tests/test_oddities.py
@@ -1,6 +1,6 @@
 import requests
 
-from utils import Commit, to_pr, seen
+from utils import Commit, to_pr
 
 
 def test_partner_merge(env):
diff --git a/runbot_merge/tests/test_provisioning.py b/runbot_merge/tests/test_provisioning.py
index db26e5e7..7cadcd52 100644
--- a/runbot_merge/tests/test_provisioning.py
+++ b/runbot_merge/tests/test_provisioning.py
@@ -179,7 +179,7 @@ def test_bulk_ops(env, port):
         'email': "coc@example.org",
         'active': False,
     }])
-    assert a.active == b.active == False
+    assert a.active is b.active is False
 
     assert provision_user(port, [
         {'email': 'bob@example.org', 'github_login': 'xyz'},
@@ -187,7 +187,7 @@ def test_bulk_ops(env, port):
     ]) == [2, 0]
     assert a.users_id
     assert b.users_id
-    assert a.active == b.active == True
+    assert a.active is b.active is True
 
 def provision_user(port, users):
     r = requests.post(f'http://localhost:{port}/runbot_merge/provision', json={

From b1af2e573a68e603c6c82393bd5382bd25e49cd8 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 10 Aug 2023 09:02:37 +0200
Subject: [PATCH 025/192] [IMP] runbot_merge: split staging heads out to join
 tables

Currently the heads of a staging (both staging heads and merged heads)
are just JSON data on the staging itself. Historically this was
convenient as the heads were mostly of use to the staging process, and
thus accessed directly through the staging essentially exclusively.

However this makes finding stagings from merged commits e.g. for
forensic research almost impossible, because querying based on
the *values* of a JSON map is expensive, and indexing it is difficult.

To make this use case more feasible, split the `heads` field into two
join tables, one for the staging heads and one for the merged heads,
this makes looking for stagings by commits much more
efficient (although the queries may not be trivial). Also add two
utility RPC methods, so that it's possible to query stagings
reasonably easily and efficiently based on a set of commits (branch
heads).

related to #768
---
 runbot_merge/__manifest__.py                  |   2 +-
 .../migrations/15.0.1.8/pre-migration.py      |   6 +
 runbot_merge/migrations/15.0.1.8/upgrade.sql  |  62 +++++
 runbot_merge/models/pull_requests.py          | 260 ++++++++++--------
 runbot_merge/security/ir.model.access.csv     |   2 +
 runbot_merge/tests/test_multirepo.py          |  31 ++-
 6 files changed, 246 insertions(+), 117 deletions(-)
 create mode 100644 runbot_merge/migrations/15.0.1.8/pre-migration.py
 create mode 100644 runbot_merge/migrations/15.0.1.8/upgrade.sql

diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py
index 3640d249..04c23c7c 100644
--- a/runbot_merge/__manifest__.py
+++ b/runbot_merge/__manifest__.py
@@ -1,6 +1,6 @@
 {
     'name': 'merge bot',
-    'version': '1.7',
+    'version': '1.8',
     'depends': ['contacts', 'website'],
     'data': [
         'security/security.xml',
diff --git a/runbot_merge/migrations/15.0.1.8/pre-migration.py b/runbot_merge/migrations/15.0.1.8/pre-migration.py
new file mode 100644
index 00000000..8bf72c44
--- /dev/null
+++ b/runbot_merge/migrations/15.0.1.8/pre-migration.py
@@ -0,0 +1,6 @@
+from pathlib import Path
+
+def migrate(cr, version):
+    sql = Path(__file__).parent.joinpath('upgrade.sql')\
+        .read_text(encoding='utf-8')
+    cr.execute(sql)
diff --git a/runbot_merge/migrations/15.0.1.8/upgrade.sql b/runbot_merge/migrations/15.0.1.8/upgrade.sql
new file mode 100644
index 00000000..d5dc0502
--- /dev/null
+++ b/runbot_merge/migrations/15.0.1.8/upgrade.sql
@@ -0,0 +1,62 @@
+CREATE TABLE runbot_merge_stagings_commits (
+    id serial NOT NULL,
+    staging_id integer not null references runbot_merge_stagings (id),
+    commit_id integer not null references runbot_merge_commit (id),
+    repository_id integer not null references runbot_merge_repository (id)
+);
+
+CREATE TABLE runbot_merge_stagings_heads (
+    id serial NOT NULL,
+    staging_id integer NOT NULL REFERENCES runbot_merge_stagings (id),
+    commit_id integer NOT NULL REFERENCES runbot_merge_commit (id),
+    repository_id integer NOT NULL REFERENCES runbot_merge_repository (id)
+);
+
+-- some of the older stagings only have the head, not the commit,
+-- add the commit
+UPDATE runbot_merge_stagings
+  SET heads = heads::jsonb || jsonb_build_object(
+    'odoo/odoo^', heads::json->'odoo/odoo',
+    'odoo/enterprise^', heads::json->'odoo/enterprise'
+  )
+  WHERE heads NOT ILIKE '%^%';
+
+-- some of the stagings have heads which don't exist in the commits table,
+-- because they never got a status from the runbot...
+-- create fake commits so we don't lose heads
+INSERT INTO runbot_merge_commit (sha, statuses, create_uid, create_date, write_uid, write_date)
+    SELECT r.value, '{}', s.create_uid, s.create_date, s.create_uid, s.create_date
+    FROM runbot_merge_stagings s,
+         json_each_text(s.heads::json) r
+ON CONFLICT DO NOTHING;
+
+CREATE TEMPORARY TABLE staging_commits (
+    id integer NOT NULL,
+    repo integer NOT NULL,
+    -- the staging head (may be a dedup, may be the same as commit)
+    head integer NOT NULL,
+    -- the staged commit
+    commit integer NOT NULL
+);
+-- the splatting works entirely off of the staged head
+-- (the one without the ^ suffix), we concat the `^` to get the corresponding
+-- merge head (the actual commit to push to the branch)
+INSERT INTO staging_commits (id, repo, head, commit)
+    SELECT s.id, re.id AS repo, h.id AS head, c.id AS commit
+    FROM runbot_merge_stagings s,
+         json_each_text(s.heads::json) r,
+         runbot_merge_commit h,
+         runbot_merge_commit c,
+         runbot_merge_repository re
+    WHERE r.key NOT ILIKE '%^'
+      AND re.name = r.key
+      AND h.sha = r.value
+      AND c.sha = s.heads::json->>(r.key || '^');
+
+INSERT INTO runbot_merge_stagings_heads (staging_id, repository_id, commit_id)
+SELECT id, repo, head FROM staging_commits;
+
+INSERT INTO runbot_merge_stagings_commits (staging_id, repository_id, commit_id)
+SELECT id, repo, commit FROM staging_commits;
+
+ALTER TABLE runbot_merge_stagings DROP COLUMN heads;
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 2873dbcf..0005704b 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -408,18 +408,19 @@ class Branch(models.Model):
         if not staged:
             return
 
-        heads = {}
+        heads = []
+        heads_map = {}
+        commits = []
         for repo, it in meta.items():
             tree = it['gh'].commit(it['head'])['tree']
             # ensures staging branches are unique and always
             # rebuilt
             r = base64.b64encode(os.urandom(12)).decode('ascii')
             trailer = ''
-            if heads:
+            if heads_map:
                 trailer = '\n'.join(
                     'Runbot-dependency: %s:%s' % (repo, h)
-                    for repo, h in heads.items()
-                    if not repo.endswith('^')
+                    for repo, h in heads_map.items()
                 )
             dummy_head = {'sha': it['head']}
             if it['head'] == original_heads[repo]:
@@ -435,28 +436,50 @@ For-Commit-Id: %s
                     'parents': [it['head']],
                 }).json()
 
-            # $repo is the head to check, $repo^ is the head to merge (they
-            # might be the same)
-            heads[repo.name + '^'] = it['head']
-            heads[repo.name] = dummy_head['sha']
-            self.env.cr.execute(
-                "INSERT INTO runbot_merge_commit (sha, to_check, statuses) "
-                "VALUES (%s, true, '{}') "
-                "ON CONFLICT (sha) DO UPDATE SET to_check=true",
-                [dummy_head['sha']]
-            )
+            # special case if the two commits are identical because otherwise
+            # postgres raises error "ensure that no rows proposed for insertion
+            # within the same command have duplicate constained values"
+            if it['head'] == dummy_head['sha']:
+                self.env.cr.execute(
+                    "INSERT INTO runbot_merge_commit (sha, to_check, statuses) "
+                    "VALUES (%s, true, '{}') "
+                    "ON CONFLICT (sha) DO UPDATE SET to_check=true "
+                    "RETURNING id",
+                    [it['head']]
+                )
+                [commit] = [head] = self.env.cr.fetchone()
+            else:
+                self.env.cr.execute(
+                    "INSERT INTO runbot_merge_commit (sha, to_check, statuses) "
+                    "VALUES (%s, false, '{}'), (%s, true, '{}') "
+                    "ON CONFLICT (sha) DO UPDATE SET to_check=true "
+                    "RETURNING id",
+                    [it['head'], dummy_head['sha']]
+                )
+                ([commit], [head]) = self.env.cr.fetchall()
+
+            heads_map[repo.name] = dummy_head['sha']
+            heads.append(fields.Command.create({
+                'repository_id': repo.id,
+                'commit_id': head,
+            }))
+            commits.append(fields.Command.create({
+                'repository_id': repo.id,
+                'commit_id': commit,
+            }))
 
         # create actual staging object
         st = self.env['runbot_merge.stagings'].create({
             'target': self.id,
             'batch_ids': [(4, batch.id, 0) for batch in staged],
-            'heads': json.dumps(heads)
+            'heads': heads,
+            'commits': commits,
         })
         # create staging branch from tmp
         token = self.project_id.github_token
         for r in self.project_id.repo_ids.having_branch(self):
             it = meta[r]
-            staging_head = heads[r.name]
+            staging_head = heads_map[r.name]
             _logger.info(
                 "%s: create staging for %s:%s at %s",
                 self.project_id.name, r.name, self.name,
@@ -1716,13 +1739,7 @@ class Commit(models.Model):
                 if pr:
                     pr._validate(st)
 
-                stagings = Stagings.search([('heads', 'ilike', c.sha)]).filtered(
-                    lambda s, h=c.sha: any(
-                        head == h
-                        for repo, head in json.loads(s.heads).items()
-                        if not repo.endswith('^')
-                    )
-                )
+                stagings = Stagings.search([('head_ids.sha', '=', c.sha)])
                 if stagings:
                     stagings._validate()
             except Exception:
@@ -1748,6 +1765,53 @@ class Commit(models.Model):
         """)
         return res
 
+class StagingCommits(models.Model):
+    _name = 'runbot_merge.stagings.commits'
+    _description = "Mergeable commits for stagings, always the actually merged " \
+                   "commit, never a uniquifier"
+    _log_access = False
+
+    staging_id = fields.Many2one('runbot_merge.stagings', required=True)
+    commit_id = fields.Many2one('runbot_merge.commit', index=True, required=True)
+    repository_id = fields.Many2one('runbot_merge.repository', required=True)
+
+    def _auto_init(self):
+        super()._auto_init()
+        # the same commit can be both head and tip (?)
+        tools.create_unique_index(
+            self.env.cr, self._table + "_unique",
+            self._table, ['staging_id', 'commit_id']
+        )
+        # there should be one head per staging per repository, unless one is a
+        # real head and one is a uniquifier head
+        tools.create_unique_index(
+            self.env.cr, self._table + "_unique_per_repo",
+            self._table, ['staging_id', 'repository_id'],
+        )
+
+class StagingHeads(models.Model):
+    _name = 'runbot_merge.stagings.heads'
+    _description = "Staging heads, may be the staging's commit or may be a " \
+                   "uniquifier (discarded on success)"
+    _log_access = False
+
+    staging_id = fields.Many2one('runbot_merge.stagings', required=True)
+    commit_id = fields.Many2one('runbot_merge.commit', index=True, required=True)
+    repository_id = fields.Many2one('runbot_merge.repository', required=True)
+
+    def _auto_init(self):
+        super()._auto_init()
+        # the same commit can be both head and tip (?)
+        tools.create_unique_index(
+            self.env.cr, self._table + "_unique",
+            self._table, ['staging_id', 'commit_id']
+        )
+        # there should be one head per staging per repository, unless one is a
+        # real head and one is a uniquifier head
+        tools.create_unique_index(
+            self.env.cr, self._table + "_unique_per_repo",
+            self._table, ['staging_id', 'repository_id'],
+        )
 class Stagings(models.Model):
     _name = _description = 'runbot_merge.stagings'
 
@@ -1771,9 +1835,10 @@ class Stagings(models.Model):
     timeout_limit = fields.Datetime(store=True, compute='_compute_timeout_limit')
     reason = fields.Text("Reason for final state (if any)")
 
-    # seems simpler than adding yet another indirection through a model
-    heads = fields.Char(required=True, help="JSON-encoded map of heads, one per repo in the project")
-    head_ids = fields.Many2many('runbot_merge.commit', compute='_compute_statuses')
+    head_ids = fields.Many2many('runbot_merge.commit', relation='runbot_merge_stagings_heads', column1='staging_id', column2='commit_id')
+    heads = fields.One2many('runbot_merge.stagings.heads', 'staging_id')
+    commit_ids = fields.Many2many('runbot_merge.commit', relation='runbot_merge_stagings_commits', column1='staging_id', column2='commit_id')
+    commits = fields.One2many('runbot_merge.stagings.commits', 'staging_id')
 
     statuses = fields.Binary(compute='_compute_statuses')
     statuses_cache = fields.Text()
@@ -1786,7 +1851,7 @@ class Stagings(models.Model):
             return super().write(vals)
 
         previously_pending = self.filtered(lambda s: s.state == 'pending')
-        super(Stagings, self).write(vals)
+        super().write(vals)
         for staging in previously_pending:
             if staging.state != 'pending':
                 super(Stagings, staging).write({
@@ -1795,7 +1860,6 @@ class Stagings(models.Model):
 
         return True
 
-
     def name_get(self):
         return [
             (staging.id, "%d (%s, %s%s)" % (
@@ -1812,33 +1876,18 @@ class Stagings(models.Model):
         """ Fetches statuses associated with the various heads, returned as
         (repo, context, state, url)
         """
-        Commits = self.env['runbot_merge.commit']
-        heads = {
-            head: repo
-            for st in self
-            for repo, head in json.loads(st.heads).items()
-            if not repo.endswith('^')
-        }
-        all_heads = Commits.search([('sha', 'in', list(heads))])
-        commits_map = {commit.sha: commit.id for commit in all_heads}
+        heads = {h.commit_id: h.repository_id for h in self.mapped('heads')}
+        all_heads = self.mapped('head_ids')
 
         for st in self:
-            commits = st.head_ids = Commits._browse(
-                self.env,
-                tuple(
-                    commits_map[h]
-                    for repo, h in json.loads(st.heads).items()
-                    if not repo.endswith('^')
-                ),
-                all_heads._prefetch_ids
-            )
             if st.statuses_cache:
                 st.statuses = json.loads(st.statuses_cache)
                 continue
 
+            commits = st.head_ids.with_prefetch(all_heads._prefetch_ids)
             st.statuses = [
                 (
-                    heads[commit.sha],
+                    heads[commit].name,
                     context,
                     status.get('state') or 'pending',
                     status.get('target_url') or ''
@@ -1865,27 +1914,17 @@ class Stagings(models.Model):
             staging.pr_ids = staging.batch_ids.prs
 
     def _validate(self):
-        Commits = self.env['runbot_merge.commit']
         for s in self:
             if s.state != 'pending':
                 continue
 
-            repos = {
-                repo.name: repo
-                for repo in self.env['runbot_merge.repository'].search([])
-                    .having_branch(s.target)
-            }
             # maps commits to the statuses they need
             required_statuses = [
-                (head, repos[repo].status_ids._for_staging(s).mapped('context'))
-                for repo, head in json.loads(s.heads).items()
-                if not repo.endswith('^')
+                (h.commit_id.sha, h.repository_id.status_ids._for_staging(s).mapped('context'))
+                for h in s.heads
             ]
             # maps commits to their statuses
-            cmap = {
-                c.sha: json.loads(c.statuses)
-                for c in Commits.search([('sha', 'in', [h for h, _ in required_statuses])])
-            }
+            cmap = {c.sha: json.loads(c.statuses) for c in s.head_ids}
 
             update_timeout_limit = False
             st = 'success'
@@ -1983,19 +2022,10 @@ class Stagings(models.Model):
             return False
 
         # try inferring which PR failed and only mark that one
-        for repo, head in json.loads(self.heads).items():
-            if repo.endswith('^'):
-                continue
+        for head in self.heads:
+            required_statuses = set(head.repository_id.status_ids._for_staging(self).mapped('context'))
 
-            required_statuses = set(
-                self.env['runbot_merge.repository']
-                    .search([('name', '=', repo)])
-                    .status_ids
-                    ._for_staging(self)
-                    .mapped('context'))
-
-            commit = self.env['runbot_merge.commit'].search([('sha', '=', head)])
-            statuses = json.loads(commit.statuses or '{}')
+            statuses = json.loads(head.commit_id.statuses or '{}')
             reason = next((
                 ctx for ctx, result in statuses.items()
                 if ctx in required_statuses
@@ -2004,10 +2034,7 @@ class Stagings(models.Model):
             if not reason:
                 continue
 
-            pr = next((
-                pr for pr in self.batch_ids.prs
-                if pr.repository.name == repo
-            ), None)
+            pr = next((pr for pr in self.batch_ids.prs if pr.repository == head.repository_id), None)
 
             status = to_status(statuses[reason])
             viewmore = ''
@@ -2016,7 +2043,7 @@ class Stagings(models.Model):
             if pr:
                 self.fail("%s%s" % (reason, viewmore), pr)
             else:
-                self.fail('%s on %s%s' % (reason, head, viewmore))
+                self.fail('%s on %s%s' % (reason, head.commit_id.sha, viewmore))
             return False
 
         # the staging failed but we don't have a specific culprit, fail
@@ -2043,7 +2070,6 @@ class Stagings(models.Model):
         project = self.target.project_id
         if self.state == 'success':
             gh = {repo.name: repo.github() for repo in project.repo_ids.having_branch(self.target)}
-            staging_heads = json.loads(self.heads)
             self.env.cr.execute('''
             SELECT 1 FROM runbot_merge_pull_requests
             WHERE id in %s
@@ -2053,7 +2079,7 @@ class Stagings(models.Model):
                 with sentry_sdk.start_span(description="merge staging") as span:
                     span.set_tag("staging", self.id)
                     span.set_tag("branch", self.target.name)
-                    self._safety_dance(gh, staging_heads)
+                    self._safety_dance(gh, self.commits)
             except exceptions.FastForwardError as e:
                 logger.warning(
                     "Could not fast-forward successful staging on %s:%s",
@@ -2100,7 +2126,7 @@ class Stagings(models.Model):
     def is_timed_out(self):
         return fields.Datetime.from_string(self.timeout_limit) < datetime.datetime.now()
 
-    def _safety_dance(self, gh, staging_heads):
+    def _safety_dance(self, gh, staging_commits: StagingCommits):
         """ Reverting updates doesn't work if the branches are protected
         (because a revert is basically a force push). So we can update
         REPO_A, then fail to update REPO_B for some reason, and we're hosed.
@@ -2117,51 +2143,69 @@ class Stagings(models.Model):
           bad. In that case, wait a bit and retry for now. A more complex
           strategy (including disabling the branch entirely until somebody
           has looked at and fixed the issue) might be necessary.
-
-        :returns: the last repo it tried to update (probably the one on which
-                  it failed, if it failed)
         """
-        # FIXME: would make sense for FFE to be richer, and contain the repo name
-        repo_name = None
         tmp_target = 'tmp.' + self.target.name
         # first force-push the current targets to all tmps
-        for repo_name in staging_heads.keys():
-            if repo_name.endswith('^'):
-                continue
+        for repo_name in staging_commits.mapped('repository_id.name'):
             g = gh[repo_name]
             g.set_ref(tmp_target, g.head(self.target.name))
-        # then attempt to FF the tmp to the staging
-        for repo_name, head in staging_heads.items():
-            if repo_name.endswith('^'):
-                continue
-            gh[repo_name].fast_forward(tmp_target, staging_heads.get(repo_name + '^') or head)
+        # then attempt to FF the tmp to the staging commits
+        for c in staging_commits:
+            gh[c.repository_id.name].fast_forward(tmp_target, c.commit_id.sha)
         # there is still a race condition here, but it's way
         # lower than "the entire staging duration"...
-        first = True
-        for repo_name, head in staging_heads.items():
-            if repo_name.endswith('^'):
-                continue
-
+        for i, c in enumerate(staging_commits):
             for pause in [0.1, 0.3, 0.5, 0.9, 0]: # last one must be 0/falsy of we lose the exception
                 try:
-                    # if the staging has a $repo^ head, merge that,
-                    # otherwise merge the regular (CI'd) head
-                    gh[repo_name].fast_forward(
+                    gh[c.repository_id.name].fast_forward(
                         self.target.name,
-                        staging_heads.get(repo_name + '^') or head
+                        c.commit_id.sha
                     )
                 except exceptions.FastForwardError:
-                    # The GH API regularly fails us. If the failure does not
-                    # occur on the first repository, retry a few times with a
-                    # little pause.
-                    if not first and pause:
+                    if i and pause:
                         time.sleep(pause)
                         continue
                     raise
                 else:
                     break
-            first = False
-        return repo_name
+
+    @api.returns('runbot_merge.stagings')
+    def for_heads(self, *heads):
+        """Returns the staging(s) with all the specified heads. Heads should
+        be unique git oids.
+        """
+        if not heads:
+            return self.browse(())
+
+        joins = ''.join(
+            f'\nJOIN runbot_merge_stagings_heads h{i} ON h{i}.staging_id = s.id'
+            f'\nJOIN runbot_merge_commit c{i} ON c{i}.id = h{i}.commit_id AND c{i}.sha = %s\n'
+            for i in range(len(heads))
+        )
+        self.env.cr.execute(f"SELECT s.id FROM runbot_merge_stagings s {joins}", heads)
+        stagings = self.browse(id for [id] in self.env.cr.fetchall())
+        stagings.check_access_rights('read')
+        stagings.check_access_rule('read')
+        return stagings
+
+    @api.returns('runbot_merge.stagings')
+    def for_commits(self, *heads):
+        """Returns the staging(s) with all the specified commits (heads which
+        have actually been merged). Commits should be unique git oids.
+        """
+        if not heads:
+            return self.browse(())
+
+        joins = ''.join(
+            f'\nJOIN runbot_merge_stagings_commits h{i} ON h{i}.staging_id = s.id'
+            f'\nJOIN runbot_merge_commit c{i} ON c{i}.id = h{i}.commit_id AND c{i}.sha = %s\n'
+            for i in range(len(heads))
+        )
+        self.env.cr.execute(f"SELECT s.id FROM runbot_merge_stagings s {joins}", heads)
+        stagings = self.browse(id for [id] in self.env.cr.fetchall())
+        stagings.check_access_rights('read')
+        stagings.check_access_rule('read')
+        return stagings
 
 class Split(models.Model):
     _name = _description = 'runbot_merge.split'
diff --git a/runbot_merge/security/ir.model.access.csv b/runbot_merge/security/ir.model.access.csv
index 006f59de..5b2e102b 100644
--- a/runbot_merge/security/ir.model.access.csv
+++ b/runbot_merge/security/ir.model.access.csv
@@ -11,6 +11,8 @@ access_runbot_merge_pull_requests_admin,Admin access to PR,model_runbot_merge_pu
 access_runbot_merge_pull_requests_tagging_admin,Admin access to tagging,model_runbot_merge_pull_requests_tagging,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_commit_admin,Admin access to commits,model_runbot_merge_commit,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_stagings_admin,Admin access to stagings,model_runbot_merge_stagings,runbot_merge.group_admin,1,1,1,1
+access_runbot_merge_stagings_heads_admin,Admin access to staging heads,model_runbot_merge_stagings_heads,runbot_merge.group_admin,1,1,1,1
+access_runbot_merge_stagings_commits_admin,Admin access to staging commits,model_runbot_merge_stagings_commits,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_stagings_cancel_admin,Admin access to cancelling stagings,model_runbot_merge_stagings_cancel,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_split_admin,Admin access to splits,model_runbot_merge_split,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_batch_admin,Admin access to batches,model_runbot_merge_batch,runbot_merge.group_admin,1,1,1,1
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index b94a1ec9..58c642fa 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -373,14 +373,29 @@ def test_sub_match(env, project, repo_a, repo_b, repo_c, config):
     a_staging = repo_a.commit('staging.master')
     b_staging = repo_b.commit('staging.master')
     c_staging = repo_c.commit('staging.master')
-    assert json.loads(st.heads) == {
-        repo_a.name: a_staging.id,
-        repo_a.name + '^': a_staging.parents[0],
-        repo_b.name: b_staging.id,
-        repo_b.name + '^': b_staging.id,
-        repo_c.name: c_staging.id,
-        repo_c.name + '^': c_staging.id,
-    }
+    assert sorted(st.head_ids.mapped('sha')) == sorted([
+        a_staging.id,
+        b_staging.id,
+        c_staging.id,
+    ])
+    s = env['runbot_merge.stagings'].for_heads(
+        a_staging.id,
+        b_staging.id,
+        c_staging.id,
+    )
+    assert s == list(st.ids)
+
+    assert sorted(st.commit_ids.mapped('sha')) == sorted([
+        a_staging.parents[0],
+        b_staging.id,
+        c_staging.id,
+    ])
+    s = env['runbot_merge.stagings'].for_commits(
+        a_staging.parents[0],
+        b_staging.id,
+        c_staging.id,
+    )
+    assert s == list(st.ids)
 
 def test_merge_fail(env, project, repo_a, repo_b, users, config):
     """ In a matched-branch scenario, if merging in one of the linked repos

From e9f7252ed12d44b1ad49f2d4f943474f0553f522 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 10 Aug 2023 15:27:20 +0200
Subject: [PATCH 026/192] [FIX] runbot_merge: sentry issue via monkeypatch

`auto_session_tracking` causes issues when not specified on the super
old version of the client which is available on ubuntu.

Also disable tracing as it seems less useful than hoped for, and I've
not been using what's been collected so far.
---
 runbot_merge/sentry.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runbot_merge/sentry.py b/runbot_merge/sentry.py
index 48bdd48e..4b87765c 100644
--- a/runbot_merge/sentry.py
+++ b/runbot_merge/sentry.py
@@ -35,7 +35,8 @@ def enable_sentry():
 def setup_sentry(dsn):
     sentry_sdk.init(
         dsn,
-        traces_sample_rate=1.0,
+        auto_session_tracking=False,
+        # traces_sample_rate=1.0,
         integrations=[
             # note: if the colorformatter is enabled, sentry gets lost
             # and classifies everything as errors because it fails to

From 4eefc980bb676cf89a946d0059b48af50f47694e Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 10 Aug 2023 16:14:33 +0200
Subject: [PATCH 027/192] [IMP] runbot_merge: logger messages

---
 runbot_merge/github.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/github.py b/runbot_merge/github.py
index 5b11033b..1461dc82 100644
--- a/runbot_merge/github.py
+++ b/runbot_merge/github.py
@@ -46,7 +46,7 @@ def _init_gh_logger():
 if odoo.netsvc._logger_init:
     _init_gh_logger()
 
-GH_LOG_PATTERN = """=> {method} /{path}{qs}{body}
+GH_LOG_PATTERN = """=> {method} {path}{qs}{body}
 
 <= {r.status_code} {r.reason}
 {headers}
@@ -222,7 +222,7 @@ class GH(object):
 
         status0 = r.status_code
         _logger.debug(
-            'ref_set(%s, %s, %s -> %s (%s)',
+            'set_ref(%s, %s, %s -> %s (%s)',
             self._repo, branch, sha, status0,
             'OK' if status0 == 200 else r.text or r.reason
         )

From 85a74a9e32e7e03e1d7958fb251cc23439d6792f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 11 Aug 2023 11:13:34 +0200
Subject: [PATCH 028/192] [ADD] runbot_merge: staging query endpoints

`/runbot_merge/stagings`
========================

This endpoint is a reverse lookup from any number of commits to a
(number of) staging(s):

- it takes a list of commit hashes as either the `commits` or the
  `heads` keyword parameter
- it then returns the stagings which have *all* these commits as
  respectively commits or heads, if providing all commits for a
  project the result should always be unique (if any)
- `commits` are the merged commits, aka the stuff which ends up in the
  actual branches
- `heads` are the staging heads, aka the commits at the tip of the
  `staging.$name` branches, those may be the same as the corresponding
  commit, or might be deduplicator commits which get discarded on
  success

`/runbot_merge/stagings/:id`
============================

Returns a list of all PRs in the staging, grouped by batch (aka PRs
which have the same label and must be merged together).

For each PR, the `repository` name, `number`, and `name` in the form
`$repository#$number` get returned.

`/runbot_merge/stagings/:id1/:id2`
==================================

Returns a list of all the *successfully merged* stagings between `id1`
and `id2`, from oldest to most recent. Individual records have the
form:

- `staging` is the id of the staging
- `prs` is the contents of the previous endpoint (a list of PRs
  grouped by batch)

`id1` *must* be lower than `id2`.

By default, this endpoint is inclusive on both ends, the
`include_from` and / or `include_to` parameters can be passed with the
`False` value to exclude the corresponding bound from the result.

Related to #768
---
 runbot_merge/controllers/__init__.py | 57 ++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 2f9fbc20..3e693824 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -15,6 +15,63 @@ from .. import utils, github
 _logger = logging.getLogger(__name__)
 
 class MergebotController(Controller):
+    @route('/runbot_merge/stagings', auth='none', type='json')
+    def stagings_for_commits(self, commits=None, heads=None):
+        Stagings = request.env(user=1)['runbot_merge.stagings'].sudo()
+        if commits:
+            stagings = Stagings.for_commits(*commits)
+        elif heads:
+            stagings = Stagings.for_heads(*heads)
+        else:
+            raise ValueError('Must receive one of "commits" or "heads" kwarg')
+
+        return stagings.ids
+
+    @route('/runbot_merge/stagings/<int:staging>', auth='none', type='json')
+    def prs_for_staging(self, staging):
+        staging = request.env(user=1)['runbot_merge.stagings'].browse(staging)
+        return [
+            batch.prs.mapped(lambda p: {
+                'name': p.display_name,
+                'repository': p.repository.name,
+                'number': p.number,
+            })
+            for batch in staging.sudo().batch_ids
+        ]
+
+    @route('/runbot_merge/stagings/<int:from_staging>/<int:to_staging>', auth='none', type='json')
+    def prs_for_staging(self, from_staging, to_staging, include_from=True, include_to=True):
+        Stagings = request.env(user=1, context={"active_test": False})['runbot_merge.stagings']
+        from_staging = Stagings.browse(from_staging)
+        to_staging = Stagings.browse(to_staging)
+        if from_staging.target != to_staging.target:
+            raise ValueError(f"Stagings must have the same target branch, found {from_staging.target.name} and {to_staging.target.name}")
+        if from_staging.id >= to_staging.id:
+            raise ValueError("first staging must be older than second staging")
+
+        stagings = Stagings.search([
+            ('target', '=', to_staging.target.id),
+            ('state', '=', 'success'),
+            ('id', '>=' if include_from else '>', from_staging.id),
+            ('id', '<=' if include_to else '<', to_staging.id),
+        ], order="id asc")
+
+        return [
+            {
+                'staging': staging.id,
+                'prs': [
+                    batch.prs.mapped(lambda p: {
+                        'name': p.display_name,
+                        'repository': p.repository.name,
+                        'number': p.number,
+                    })
+                    for batch in staging.batch_ids
+                ]
+            }
+            for staging in stagings
+        ]
+
+
     @route('/runbot_merge/hooks', auth='none', type='json', csrf=False, methods=['POST'])
     def index(self):
         req = request.httprequest

From 7348e4d7a4d3565433423cc2d91307d40fd46d35 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 11 Aug 2023 11:50:38 +0200
Subject: [PATCH 029/192] [IMP] runbot_merge: ensure at least 1s between
 mutating GH calls

Mostly a temporary safety feature after the events of 07-31: it's
still not clear whether that was a one-off issue or a change in
policy (I was not able to reproduce locally even doing several
set_refs a second) and the gh support is not super talkative, but it
probably doesn't hurt to commit the workaround until #247 gets
implemented.

On 2023-07-31, around 08:30 UTC, `set_ref` started failing, a lot
(although oddly enough not continuously), with the unhelpful message
that

> 422: Reference cannot be updated

This basically broke all stagings, until a workaround was implemented
by adding a 1s sleep before `set_ref` to ensure no more than 1
`set_ref` per second, which kinda sorta has been the github
recommendation forever but had never been an issue
before. Contributing to this suspicion is that in late 2022, the
documentation of error 422 on `PATCH git/refs/{ref}` was updated to:

> Validation failed, or the endpoint has been spammed.

Still would be nice if GH was clear about it and sent a 429 instead.

Technically the recommendation is:

> If you're making a large number of POST, PATCH, PUT, or DELETE
> requests for a single user or client ID, wait at least one second
> between each request.

So... actually implement that. On a per-worker basis as for the most
part these are serial processes (e.g. crons), we can still get above
the rate limit due to concurrent crons but it should be less likely.

Also take `Retry-After` in account, can't hurt, though we're supposed
to retry just the request rather than abort the entire thing. Maybe a
future update can improve this handling.

Would also be nice to take `X-RateLimit` in account, although that's
supposed to apply to *all* requests so we'd need a second separate
timestamp to track it. Technically that's probably also the case for
`Retry-After`. And fixing #247 should cut down drastically on the API
calls traffic as staging is a very API-intensive process, especially
with the sanity checks we had to add, these days we might be at 4
calls per commit per PR, and up to 80 PRs/staging (5 repositories and
16 batches per staging), with 13 live branches (though realistically
only 6-7 have significant traffic, and only 1~2 get close to filling
their staging slots).
---
 runbot_merge/github.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/runbot_merge/github.py b/runbot_merge/github.py
index 1461dc82..ed3950b0 100644
--- a/runbot_merge/github.py
+++ b/runbot_merge/github.py
@@ -6,6 +6,7 @@ import logging.handlers
 import os
 import pathlib
 import pprint
+import time
 import unicodedata
 
 import requests
@@ -57,6 +58,7 @@ class GH(object):
     def __init__(self, token, repo):
         self._url = 'https://api.github.com'
         self._repo = repo
+        self._last_update = 0
         session = self._session = requests.Session()
         session.headers['Authorization'] = 'token {}'.format(token)
         session.headers['Accept'] = 'application/vnd.github.symmetra-preview+json'
@@ -103,8 +105,16 @@ class GH(object):
         """
         :type check: bool | dict[int:Exception]
         """
+        if method.casefold() != 'get':
+            to_sleep = 1. - (time.time() - self._last_update)
+            if to_sleep > 0:
+                time.sleep(to_sleep)
+
         path = f'/repos/{self._repo}/{path}'
         r = self._session.request(method, self._url + path, params=params, json=json)
+        if method.casefold() != 'get':
+            self._last_update = time.time() + int(r.headers.get('Retry-After', 0))
+
         self._log_gh(_gh, r)
         if check:
             try:

From 62fb880a45151e02e682edbef0367a34928e5d14 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 14 Aug 2023 08:00:56 +0200
Subject: [PATCH 030/192] [FIX] forwardport: sync outstandings notification
 with page

In 81ce4ea02b76af4dcc859739985a41ff8c073b6e the delta for PRs being
listed in the `/forwardport/outstanding` page was increased from 3
days to 7 (1 week), however the warning box in the home page still
used the old cutoffs leading to

An inconsistency between the two and an effective severe overcounting,
as the reason why the cutoff was increased to a week is forward ports
can take a while or the author / reviewer can be a touch busy at end
of week, so 3~4 days are routine when a PR is merged on thursday or
friday (and even worse if there's bank holidays in the mix).
---
 forwardport/data/views.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/forwardport/data/views.xml b/forwardport/data/views.xml
index 08ef8177..c64f9bb6 100644
--- a/forwardport/data/views.xml
+++ b/forwardport/data/views.xml
@@ -11,11 +11,11 @@
             <t t-if="env['runbot_merge.pull_requests'].check_access_rights('read', False)">
                 <t t-set="outstanding" t-value="env['runbot_merge.pull_requests'].search_count([
                     ('state', 'in', ['opened', 'validated', 'approved', 'ready', 'error']),
-                    ('source_id.merge_date', '&lt;', datetime.datetime.now() - relativedelta(days=3)),
+                    ('source_id.merge_date', '&lt;', datetime.datetime.now() - relativedelta(days=7)),
                 ])"/>
                 <div t-if="outstanding != 0" class="alert col-md-12 alert-warning mb-0">
                     <a href="/forwardport/outstanding">
-                        <t t-esc="outstanding"/> outstanding forward-ports
+                        <t t-esc="outstanding"/> outstanding forward-ports (>1 week)
                     </a>
                 </div>
             </t>

From 90961b99c90c801ab1b2ec177d9b9fc6b8eb448a Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 14 Aug 2023 09:27:43 +0200
Subject: [PATCH 031/192] [ADD] *: changelog entries I forgot

Can't hurt to *have* them.
---
 forwardport/changelog/2023-08/outstanding.md           | 10 ++++++++++
 runbot_merge/changelog/2023-08/opts.md                 |  6 ++++++
 .../changelog/2023-08/staging-reverse-index.md         |  6 ++++++
 runbot_merge/changelog/2023-08/stagings-to-prs.md      |  5 +++++
 4 files changed, 27 insertions(+)
 create mode 100644 forwardport/changelog/2023-08/outstanding.md
 create mode 100644 runbot_merge/changelog/2023-08/opts.md
 create mode 100644 runbot_merge/changelog/2023-08/staging-reverse-index.md
 create mode 100644 runbot_merge/changelog/2023-08/stagings-to-prs.md

diff --git a/forwardport/changelog/2023-08/outstanding.md b/forwardport/changelog/2023-08/outstanding.md
new file mode 100644
index 00000000..2de5ef2c
--- /dev/null
+++ b/forwardport/changelog/2023-08/outstanding.md
@@ -0,0 +1,10 @@
+IMP: outstandings page
+
+- increased time-before-outstanding from 3 to 7 days, as 3~4 days is common in
+  normal operations, especially when merging from very low branches were
+  forward-porting may take a while
+- improved performances by optimising fetching & filtering
+- added counts to the main listing for clarity (instead of hiding them in a
+  popover)
+- added the *original authors* for the outstanding forward ports
+- added ability to filter by team, if such are configured
diff --git a/runbot_merge/changelog/2023-08/opts.md b/runbot_merge/changelog/2023-08/opts.md
new file mode 100644
index 00000000..84cc99b0
--- /dev/null
+++ b/runbot_merge/changelog/2023-08/opts.md
@@ -0,0 +1,6 @@
+IMP: optimize home page
+
+An unnecessary deopt and a few opportunities were found and fixed in the home
+page / main dashboard, a few improvements have been implemented which should
+significantly lower the number of SQL queries and the time needed to generate
+the page.
diff --git a/runbot_merge/changelog/2023-08/staging-reverse-index.md b/runbot_merge/changelog/2023-08/staging-reverse-index.md
new file mode 100644
index 00000000..e276ed40
--- /dev/null
+++ b/runbot_merge/changelog/2023-08/staging-reverse-index.md
@@ -0,0 +1,6 @@
+ADD: stagings reverse index (from commits)
+
+Finding out the commits from a staging is not great but it's easy enough, the
+reverse was difficult and very inefficient. Splat out the "heads" JSON field
+into two join tables, and provide both ORM methods and a JSON endpoint to
+lookup stagings based on their commits.
diff --git a/runbot_merge/changelog/2023-08/stagings-to-prs.md b/runbot_merge/changelog/2023-08/stagings-to-prs.md
new file mode 100644
index 00000000..0deb8aa8
--- /dev/null
+++ b/runbot_merge/changelog/2023-08/stagings-to-prs.md
@@ -0,0 +1,5 @@
+IMP: added quick jump from staging to PR in the backend
+
+In the backend, going through the batches to reach a PR is really not
+convenient, directly displaying both github URL and frontend URL for each PR
+makes jumping around much easier.

From 9b5bb338b4155daa593f47a75b689e84e95aa159 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 24 Aug 2023 10:47:16 +0200
Subject: [PATCH 032/192] [REM] runbot_merge: status compatibility functions

When I updated the status storage (including `previous_failure`) for
some reason I didn't just migrate from the old to the new format, and
added bridge functions instead.

This is not really necessary (or useful), so convert all the legacy
data and remove the conversion helpers.

Relates to #802
---
 .../migrations/15.0.1.9/pre-migration.py      | 36 +++++++++++++++
 runbot_merge/models/pull_requests.py          | 46 ++++---------------
 runbot_merge/tests/test_basic.py              |  6 ---
 3 files changed, 46 insertions(+), 42 deletions(-)
 create mode 100644 runbot_merge/migrations/15.0.1.9/pre-migration.py

diff --git a/runbot_merge/migrations/15.0.1.9/pre-migration.py b/runbot_merge/migrations/15.0.1.9/pre-migration.py
new file mode 100644
index 00000000..c8c31582
--- /dev/null
+++ b/runbot_merge/migrations/15.0.1.9/pre-migration.py
@@ -0,0 +1,36 @@
+from psycopg2.extras import execute_values
+
+
+def migrate(cr, version):
+    # Drop all legacy style "previous failures": this is for PRs
+    # several years old so almost certainly long irrelevant, and it
+    # allows removing the workaround for them. Legacy style has the
+    # `state`, `target`, `description` keys at the toplevel while new
+    # style is like commit statuses, with the contexts at the toplevel
+    # and the status info below.
+    cr.execute("""
+UPDATE runbot_merge
+   SET previous_failure = '{}'
+ WHERE previous_failure::jsonb ? 'state'
+""")
+
+    # Getting this into postgres native manipulations seems a bit too
+    # complicated, and not really necessary.
+    cr.execute("""
+SELECT id, statuses::json
+  FROM runbot_merge_commit
+ WHERE jsonb_path_match(statuses::jsonb, '$.*.type() != "object"')
+""")
+    updated = [
+        (id, {
+            k: {'state': r, 'target_url': None, 'description': None}
+            for k, r in st.items()
+        })
+        for id, st in cr.fetchall()
+    ]
+    execute_values(cr._obj, """
+UPDATE runbot_merge_commit c
+   SET c.statuses = data.st
+  FROM (VALUES %s) AS data (id, st)
+ WHERE c.id = data.id
+""", updated)
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 0005704b..ed50d02b 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -741,7 +741,7 @@ class PullRequests(models.Model):
 
             st = 'success'
             for ci in pr.repository.status_ids._for_pr(pr):
-                v = state_(statuses, ci.context) or 'pending'
+                v = (statuses.get(ci.context) or {'state': 'pending'})['state']
                 if v in ('error', 'failure'):
                     st = 'failure'
                     break
@@ -1042,14 +1042,15 @@ class PullRequests(models.Model):
 
             success = True
             for ci in required:
-                st = state_(sts, ci) or 'pending'
-                if st == 'success':
+                status = sts.get(ci) or {'state': 'pending'}
+                result = status['state']
+                if result == 'success':
                     continue
 
                 success = False
-                if st in ('error', 'failure'):
+                if result in ('error', 'failure'):
                     failed |= pr
-                    pr._notify_ci_new_failure(ci, to_status(sts.get(ci.strip(), 'pending')))
+                    pr._notify_ci_new_failure(ci, status)
             if success:
                 oldstate = pr.state
                 if oldstate == 'opened':
@@ -1060,8 +1061,6 @@ class PullRequests(models.Model):
 
     def _notify_ci_new_failure(self, ci, st):
         prev = json.loads(self.previous_failure)
-        if prev.get('state'): # old-style previous-failure
-            prev = {ci: prev}
         if not any(self._statuses_equivalent(st, v) for v in prev.values()):
             prev[ci] = st
             self.previous_failure = json.dumps(prev)
@@ -1893,8 +1892,7 @@ class Stagings(models.Model):
                     status.get('target_url') or ''
                 )
                 for commit in commits
-                for context, st in json.loads(commit.statuses).items()
-                for status in [to_status(st)]
+                for context, status in json.loads(commit.statuses).items()
             ]
 
     # only depend on staged_at as it should not get modified, but we might
@@ -1930,7 +1928,7 @@ class Stagings(models.Model):
             st = 'success'
             for head, reqs in required_statuses:
                 statuses = cmap.get(head) or {}
-                for v in map(lambda n: state_(statuses, n), reqs):
+                for v in map(lambda n: statuses.get(n, {}).get('state'), reqs):
                     if st == 'failure' or v in ('error', 'failure'):
                         st = 'failure'
                     elif v is None:
@@ -2029,14 +2027,14 @@ class Stagings(models.Model):
             reason = next((
                 ctx for ctx, result in statuses.items()
                 if ctx in required_statuses
-                if to_status(result).get('state') in ('error', 'failure')
+                if result.get('state') in ('error', 'failure')
             ), None)
             if not reason:
                 continue
 
             pr = next((pr for pr in self.batch_ids.prs if pr.repository == head.repository_id), None)
 
-            status = to_status(statuses[reason])
+            status = statuses[reason]
             viewmore = ''
             if status.get('target_url'):
                 viewmore = ' (view more at %(target_url)s)' % status
@@ -2347,30 +2345,6 @@ class FetchJob(models.Model):
             if commit:
                 self.env.cr.commit()
 
-# The commit (and PR) statuses was originally a map of ``{context:state}``
-# however it turns out to clarify error messages it'd be useful to have
-# a bit more information e.g. a link to the CI's build info on failure and
-# all that. So the db-stored statuses are now becoming a map of
-# ``{ context: {state, target_url, description } }``. The issue here is
-# there's already statuses stored in the db so we need to handle both
-# formats, hence these utility functions)
-def state_(statuses, name):
-    """ Fetches the status state """
-    name = name.strip()
-    v = statuses.get(name)
-    if isinstance(v, dict):
-        return v.get('state')
-    return v
-def to_status(v):
-    """ Converts old-style status values (just a state string) to new-style
-    (``{state, target_url, description}``)
-
-    :type v: str | dict
-    :rtype: dict
-    """
-    if isinstance(v, dict):
-        return v
-    return {'state': v, 'target_url': None, 'description': None}
 
 refline = re.compile(rb'([\da-f]{40}) ([^\0\n]+)(\0.*)?\n?$')
 ZERO_REF = b'0'*40
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index ccc8e36f..1194283e 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -58,13 +58,7 @@ def test_trivial_flow(env, repo, page, users, config):
 
     with repo:
         repo.post_status(c1, 'success', 'legal/cla')
-    # rewrite status payload in old-style to ensure it does not break
-    c = env['runbot_merge.commit'].search([('sha', '=', c1)])
-    c.statuses = json.dumps({k: v['state'] for k, v in json.loads(c.statuses).items()})
-
-    with repo:
         repo.post_status(c1, 'success', 'ci/runbot')
-
     env.run_crons()
     assert pr_id.state == 'validated'
 

From 0826b3484bae4189402e78159a0384b59f5b1683 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 25 Aug 2023 09:51:37 +0200
Subject: [PATCH 033/192] [ADD] runbot_merge: view improvements

- add formatting for a bunch of backend objects
- add cross-links in order to use toplevel navigation between objects
  e.g. project -> branch -> staging -> PR with breadcrumbs instead of
  shitty dialog boxes

Relates to #802
---
 runbot_merge/models/pull_requests.py        |  4 +
 runbot_merge/views/mergebot.xml             | 98 +++++++++++++++++++--
 runbot_merge/views/runbot_merge_project.xml |  2 +
 3 files changed, 96 insertions(+), 8 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index ed50d02b..ba8ac7f7 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1716,6 +1716,10 @@ class Commit(models.Model):
     statuses = fields.Char(help="json-encoded mapping of status contexts to states", default="{}")
     to_check = fields.Boolean(default=False)
 
+    head_ids = fields.Many2many('runbot_merge.stagings', relation='runbot_merge_stagings_heads', column2='staging_id', column1='commit_id')
+    commit_ids = fields.Many2many('runbot_merge.stagings', relation='runbot_merge_stagings_commits', column2='staging_id', column1='commit_id')
+
+
     def create(self, values):
         values['to_check'] = True
         r = super(Commit, self).create(values)
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index ab5c8ee4..a05fcc57 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -20,6 +20,7 @@
                     <separator string="Required Statuses"/>
                     <field name="status_ids">
                         <tree editable="bottom">
+                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
                             <field name="context"/>
                             <field name="branch_filter"/>
                             <field name="prs"/>
@@ -30,6 +31,41 @@
             </form>
         </field>
     </record>
+    <record id="runbot_merge_branch_form" model="ir.ui.view">
+        <field name="name">Branch Form</field>
+        <field name="model">runbot_merge.branch</field>
+        <field name="arch" type="xml">
+            <form>
+                <sheet>
+                    <div class="oe_title">
+                        <h1><field name="name"/></h1>
+                    </div>
+                    <group>
+                        <group>
+                            <field name="project_id" readonly="1"/>
+                            <field name="sequence" readonly="1"/>
+                        </group>
+                        <group>
+                            <field name="active"/>
+                            <field name="staging_enabled"/>
+                        </group>
+                    </group>
+                    <separator string="Stagings"/>
+                    <group>
+                        <field name="active_staging_id"/>
+                    </group>
+                    <field name="staging_ids" nolabel="1" readonly="1">
+                        <tree default_order="staged_at desc">
+                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                            <field name="id"/>
+                            <field name="staged_at"/>
+                            <field name="state"/>
+                        </tree>
+                    </field>
+                </sheet>
+            </form>
+        </field>
+    </record>
 
     <record id="runbot_merge_action_projects" model="ir.actions.act_window">
         <field name="name">Projects</field>
@@ -124,6 +160,7 @@
                         <group colspan="4" string="Delegates">
                             <field name="delegates" nolabel="1">
                                 <tree>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
                                     <field name="name"/>
                                     <field name="github_login"/>
                                 </tree>
@@ -189,17 +226,28 @@
                             <field name="staged_at"/>
                         </group>
                     </group>
-                    <group string="Heads">
-                        <field name="head_ids" colspan="4" nolabel="1">
-                            <tree>
-                                <field name="sha"/>
-                                <field name="statuses"/>
-                            </tree>
-                        </field>
+                    <group>
+                        <group string="Heads">
+                            <field name="head_ids" colspan="2" nolabel="1" readonly="1">
+                                <tree>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                    <field name="sha"/>
+                                </tree>
+                            </field>
+                        </group>
+                        <group string="Commits">
+                            <field name="commit_ids" colspan="2" nolabel="1" readonly="1">
+                                <tree>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                    <field name="sha"/>
+                                </tree>
+                            </field>
+                        </group>
                     </group>
                     <group string="PRs">
                         <field name="pr_ids" colspan="4" nolabel="1" readonly="1">
                             <tree>
+                                <button type="object" name="get_formview_action" icon="fa-external-link"/>
                                 <field name="display_name"/>
                                 <field name="github_url" widget="url"/>
                                 <field name="url" widget="url"/>
@@ -209,6 +257,7 @@
                     <group string="Batches">
                         <field name="batch_ids" colspan="4" nolabel="1" readonly="1">
                             <tree>
+                                <button type="object" name="get_formview_action" icon="fa-external-link"/>
                                 <field name="prs" widget="many2many_tags"/>
                             </tree>
                         </field>
@@ -229,10 +278,43 @@
         <field name="arch" type="xml">
             <tree>
                 <field name="sha"/>
-                <field name="statuses"/>
             </tree>
         </field>
     </record>
+    <record id="runbot_merge_commits_form" model="ir.ui.view">
+        <field name="name">commits form</field>
+        <field name="model">runbot_merge.commit</field>
+        <field name="arch" type="xml">
+            <form>
+                <sheet>
+                    <div class="oe_title">
+                        <h1><field name="sha"/></h1>
+                    </div>
+                    <field name="statuses" widget="json"/>
+                    <separator string="Stagings (commits)"/>
+                    <field name="commit_ids" nolabel="1" readonly="1">
+                        <tree>
+                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                            <field name="target"/>
+                            <field name="id"/>
+                            <field name="staged_at"/>
+                            <field name="state"/>
+                        </tree>
+                    </field>
+                    <separator string="Stagings (heads)"/>
+                    <field name="head_ids" nolabel="1" readonly="1">
+                        <tree>
+                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                            <field name="target"/>
+                            <field name="id"/>
+                            <field name="staged_at"/>
+                            <field name="state"/>
+                        </tree>
+                    </field>
+                </sheet>
+            </form>
+        </field>
+    </record>
 
     <menuitem name="Mergebot" id="runbot_merge_menu"/>
         <menuitem name="Projects" id="runbot_merge_menu_project"
diff --git a/runbot_merge/views/runbot_merge_project.xml b/runbot_merge/views/runbot_merge_project.xml
index 544f9cf9..67487e38 100644
--- a/runbot_merge/views/runbot_merge_project.xml
+++ b/runbot_merge/views/runbot_merge_project.xml
@@ -46,6 +46,7 @@
                     <field name="repo_ids">
                         <tree>
                             <field name="sequence" widget="handle"/>
+                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
                             <field name="name"/>
                             <field name="branch_filter"/>
                             <field name="status_ids" widget="many2many_tags"/>
@@ -55,6 +56,7 @@
                     <field name="branch_ids">
                         <tree editable="bottom" decoration-muted="not active">
                             <field name="sequence" widget="handle" />
+                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
                             <field name="name"/>
                             <field name="active" widget="boolean_toggle"/>
                             <field name="staging_enabled" widget="boolean_toggle"/>

From 7bca6f0bd7a180ec027f892779fe91dc2fcc8b19 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 25 Aug 2023 10:59:48 +0200
Subject: [PATCH 034/192] [ADD] runbot_merge: allow resolving commits by sha

`_rec_name = 'sha'` means name_search and cross-model searches will
work much better.

Relates to #802
---
 runbot_merge/models/pull_requests.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index ba8ac7f7..fe9ad9e6 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1711,6 +1711,7 @@ class Commit(models.Model):
     statuses only, by PR pushes, by branch updates, ...
     """
     _name = _description = 'runbot_merge.commit'
+    _rec_name = 'sha'
 
     sha = fields.Char(required=True)
     statuses = fields.Char(help="json-encoded mapping of status contexts to states", default="{}")

From 9de18de4546591a6943fd2ec11a6f91188e190cb Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 16 Aug 2023 14:37:19 +0200
Subject: [PATCH 035/192] [CHG] *: move repo cache from forwardbot to mergebot

If the stagings are going to be created locally (via a git working
copy rather than the github API), the mergebot part needs to have
access to the cache, so move the cache over. Also move the maintenance
cron.

In an extermely minor way, this prefigures the (hopeful) eventual
merging of the ~~planes~~ modules.
---
 forwardport/__manifest__.py                   |   2 +-
 forwardport/data/crons.xml                    |  13 --
 forwardport/data/security.xml                 |   9 --
 .../migrations/15.0.1.3/pre-migration.py      |   9 ++
 forwardport/models/forwardport.py             |  54 +------
 forwardport/models/project.py                 | 133 ++----------------
 runbot_merge/__manifest__.py                  |   1 +
 runbot_merge/git.py                           | 124 ++++++++++++++++
 runbot_merge/models/__init__.py               |   1 +
 runbot_merge/models/crons/__init__.py         |   1 +
 runbot_merge/models/crons/git_maintenance.py  |  37 +++++
 runbot_merge/models/crons/git_maintenance.xml |  26 ++++
 12 files changed, 214 insertions(+), 196 deletions(-)
 create mode 100644 forwardport/migrations/15.0.1.3/pre-migration.py
 create mode 100644 runbot_merge/git.py
 create mode 100644 runbot_merge/models/crons/__init__.py
 create mode 100644 runbot_merge/models/crons/git_maintenance.py
 create mode 100644 runbot_merge/models/crons/git_maintenance.xml

diff --git a/forwardport/__manifest__.py b/forwardport/__manifest__.py
index 3ece49af..1f59983e 100644
--- a/forwardport/__manifest__.py
+++ b/forwardport/__manifest__.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 {
     'name': 'forward port bot',
-    'version': '1.2',
+    'version': '1.3',
     'summary': "A port which forward ports successful PRs.",
     'depends': ['runbot_merge'],
     'data': [
diff --git a/forwardport/data/crons.xml b/forwardport/data/crons.xml
index 1360914c..02d2be1e 100644
--- a/forwardport/data/crons.xml
+++ b/forwardport/data/crons.xml
@@ -42,17 +42,4 @@
         <field name="numbercall">-1</field>
         <field name="doall" eval="False"/>
     </record>
-
-    <record model="ir.cron" id="maintenance">
-        <field name="name">Maintenance of repo cache</field>
-        <field name="model_id" ref="model_forwardport_maintenance"/>
-        <field name="state">code</field>
-        <field name="code">model._run()</field>
-        <!-- run sunday morning as it can take a while, unlikely someone will need to forward-port stuff at that point -->
-        <field name="nextcall" eval="datetime.utcnow() + relativedelta(weekday=6, hour=2, minute=0, second=0, microsecond=0)"/>
-        <field name="interval_number">1</field>
-        <field name="interval_type">weeks</field>
-        <field name="numbercall">-1</field>
-        <field name="doall" eval="False"/>
-    </record>
 </odoo>
diff --git a/forwardport/data/security.xml b/forwardport/data/security.xml
index 424e7991..99548e0c 100644
--- a/forwardport/data/security.xml
+++ b/forwardport/data/security.xml
@@ -43,13 +43,4 @@
         <field name="perm_write">0</field>
         <field name="perm_unlink">0</field>
     </record>
-
-    <record id="access_forwardport_maintenance" model="ir.model.access">
-        <field name="name">Access to maintenance is useless</field>
-        <field name="model_id" ref="model_forwardport_maintenance"/>
-        <field name="perm_read">0</field>
-        <field name="perm_create">0</field>
-        <field name="perm_write">0</field>
-        <field name="perm_unlink">0</field>
-    </record>
 </odoo>
diff --git a/forwardport/migrations/15.0.1.3/pre-migration.py b/forwardport/migrations/15.0.1.3/pre-migration.py
new file mode 100644
index 00000000..ab2fbdd4
--- /dev/null
+++ b/forwardport/migrations/15.0.1.3/pre-migration.py
@@ -0,0 +1,9 @@
+import pathlib
+
+from odoo.tools.appdirs import user_cache_dir
+
+
+def migrate(_cr, _version):
+    # avoid needing to re-clone our repo unnecessarily
+    pathlib.Path(user_cache_dir('forwardport')).rename(
+        pathlib.Path(user_cache_dir('mergebot')))
diff --git a/forwardport/models/forwardport.py b/forwardport/models/forwardport.py
index 6e3e0c24..3bddf74d 100644
--- a/forwardport/models/forwardport.py
+++ b/forwardport/models/forwardport.py
@@ -1,20 +1,15 @@
 # -*- coding: utf-8 -*-
 import logging
-import pathlib
-
-import sentry_sdk
-
-import resource
-import subprocess
 import uuid
 from contextlib import ExitStack
 from datetime import datetime, timedelta
 
+import sentry_sdk
 from dateutil import relativedelta
 
 from odoo import fields, models
+from odoo.addons.runbot_merge import git
 from odoo.addons.runbot_merge.github import GH
-from odoo.tools.appdirs import user_cache_dir
 
 # how long a merged PR survives
 MERGE_AGE = relativedelta.relativedelta(weeks=2)
@@ -177,7 +172,7 @@ class UpdateQueue(models.Model, Queue):
                 # doesn't propagate revisions fast enough so on the next loop we
                 # can't find the revision we just pushed
                 dummy_branch = str(uuid.uuid4())
-                ref = previous._get_local_directory()
+                ref = git.get_local(previous.repository, 'fp_github')
                 working_copy.push(ref._directory, f'{new_head}:refs/heads/{dummy_branch}')
                 ref.branch('--delete', '--force', dummy_branch)
                 # then update the child's branch to the new head
@@ -261,46 +256,3 @@ class DeleteBranches(models.Model, Queue):
                 r.json()
             )
         _deleter.info('✔ deleted branch %s of PR %s', self.pr_id.label, self.pr_id.display_name)
-
-_gc = _logger.getChild('maintenance')
-def _bypass_limits():
-    """Allow git to go beyond the limits set for Odoo.
-
-    On large repositories, git gc can take a *lot* of memory (especially with
-    `--aggressive`), if the Odoo limits are too low this can prevent the gc
-    from running, leading to a lack of packing and a massive amount of cruft
-    accumulating in the working copy.
-    """
-    resource.setrlimit(resource.RLIMIT_AS, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
-
-class GC(models.TransientModel):
-    _name = 'forwardport.maintenance'
-    _description = "Weekly maintenance of... cache repos?"
-
-    def _run(self):
-        # lock out the forward port cron to avoid concurrency issues while we're
-        # GC-ing it: wait until it's available, then SELECT FOR UPDATE it,
-        # which should prevent cron workers from running it
-        fp_cron = self.env.ref('forwardport.port_forward')
-        self.env.cr.execute("""
-            SELECT 1 FROM ir_cron
-            WHERE id = %s
-            FOR UPDATE
-        """, [fp_cron.id])
-
-        repos_dir = pathlib.Path(user_cache_dir('forwardport'))
-        # run on all repos with a forwardport target (~ forwardport enabled)
-        for repo in self.env['runbot_merge.repository'].search([('fp_remote_target', '!=', False)]):
-            repo_dir = repos_dir / repo.name
-            if not repo_dir.is_dir():
-                continue
-
-            _gc.info('Running maintenance on %s', repo.name)
-            r = subprocess.run(
-                ['git', '--git-dir', repo_dir, 'gc', '--aggressive', '--prune=now'],
-                stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                encoding='utf-8',
-                preexec_fn = _bypass_limits,
-            )
-            if r.returncode:
-                _gc.warning("Maintenance failure (status=%d):\n%s", r.returncode, r.stdout)
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index cf60f68a..573d0086 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -20,23 +20,22 @@ import json
 import logging
 import operator
 import os
-import pathlib
 import re
 import subprocess
 import tempfile
 import typing
+from pathlib import Path
 
 import dateutil.relativedelta
 import requests
 
-import resource
 from odoo import _, models, fields, api
 from odoo.osv import expression
 from odoo.exceptions import UserError
 from odoo.tools.misc import topological_sort, groupby
 from odoo.tools.sql import reverse_order
 from odoo.tools.appdirs import user_cache_dir
-from odoo.addons.runbot_merge import utils
+from odoo.addons.runbot_merge import git, utils
 from odoo.addons.runbot_merge.models.pull_requests import RPLUS
 
 footer = '\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n'
@@ -838,14 +837,6 @@ class PullRequests(models.Model):
             b.prs[0]._schedule_fp_followup()
         return b
 
-    @property
-    def _source_url(self):
-        return 'https://{}:{}@github.com/{}'.format(
-            self.repository.project_id.fp_github_name or '',
-            self.repository.project_id.fp_github_token,
-            self.repository.name,
-        )
-
     def _create_fp_branch(self, target_branch, fp_branch_name, cleanup):
         """ Creates a forward-port for the current PR to ``target_branch`` under
         ``fp_branch_name``.
@@ -865,25 +856,26 @@ class PullRequests(models.Model):
             "Forward-porting %s (%s) to %s",
             self.display_name, root.display_name, target_branch.name
         )
-        source = self._get_local_directory()
+        source = git.get_local(self.repository, 'fp_github')
         r = source.with_config(stdout=subprocess.PIPE, stderr=subprocess.STDOUT).fetch()
         logger.info("Updated cache repo %s:\n%s", source._directory, r.stdout.decode())
 
         logger.info("Create working copy...")
+        cache_dir = user_cache_dir('forwardport')
+        # PullRequest.display_name is `owner/repo#number`, so `owner` becomes a
+        # directory, `TemporaryDirectory` only creates the leaf, so we need to
+        # make sure `owner` exists in `cache_dir`.
+        Path(cache_dir, root.repository.name).parent.mkdir(parents=True, exist_ok=True)
         working_copy = source.clone(
             cleanup.enter_context(
                 tempfile.TemporaryDirectory(
-                    prefix='%s-to-%s-' % (
-                        root.display_name,
-                        target_branch.name
-                    ),
-                    dir=user_cache_dir('forwardport')
-                )),
+                    prefix=f'{root.display_name}-to-{target_branch.name}',
+                    dir=cache_dir)),
             branch=target_branch.name
         )
 
         r = working_copy.with_config(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) \
-            .fetch(self._source_url, root.head)
+            .fetch(git.source_url(self.repository, 'fp_github'), root.head)
         logger.info(
             "Fetched head of %s into %s:\n%s",
             root.display_name,
@@ -1078,26 +1070,6 @@ stderr:
         # don't stringify so caller can still perform alterations
         return msg
 
-    def _get_local_directory(self):
-        repos_dir = pathlib.Path(user_cache_dir('forwardport'))
-        repos_dir.mkdir(parents=True, exist_ok=True)
-        repo_dir = repos_dir / self.repository.name
-
-        if repo_dir.is_dir():
-            return git(repo_dir)
-        else:
-            _logger.info("Cloning out %s to %s", self.repository.name, repo_dir)
-            subprocess.run(['git', 'clone', '--bare', self._source_url, str(repo_dir)], check=True)
-            # bare repos don't have fetch specs by default, and fetching *into*
-            # them is a pain in the ass, configure fetch specs so `git fetch`
-            # works properly
-            repo = git(repo_dir)
-            repo.config('--add', 'remote.origin.fetch', '+refs/heads/*:refs/heads/*')
-            # negative refspecs require git 2.29
-            repo.config('--add', 'remote.origin.fetch', '^refs/heads/tmp.*')
-            repo.config('--add', 'remote.origin.fetch', '^refs/heads/staging.*')
-            return repo
-
     def _outstanding(self, cutoff):
         """ Returns "outstanding" (unmerged and unclosed) forward-ports whose
         source was merged before ``cutoff`` (all of them if not provided).
@@ -1161,89 +1133,6 @@ class Feedback(models.Model):
 
     token_field = fields.Selection(selection_add=[('fp_github_token', 'Forwardport Bot')])
 
-ALWAYS = ('gc.auto=0', 'maintenance.auto=0')
-
-def _bypass_limits():
-    resource.setrlimit(resource.RLIMIT_AS, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
-
-def git(directory): return Repo(directory, check=True)
-class Repo:
-    def __init__(self, directory, **config):
-        self._directory = str(directory)
-        config.setdefault('stderr', subprocess.PIPE)
-        self._config = config
-        self._params = ()
-        self._opener = subprocess.run
-
-    def __getattr__(self, name):
-        return GitCommand(self, name.replace('_', '-'))
-
-    def _run(self, *args, **kwargs):
-        opts = {**self._config, **kwargs}
-        args = ('git', '-C', self._directory)\
-            + tuple(itertools.chain.from_iterable(('-c', p) for p in self._params + ALWAYS))\
-            + args
-        try:
-            return self._opener(args, preexec_fn=_bypass_limits, **opts)
-        except subprocess.CalledProcessError as e:
-            stream = e.stderr if e.stderr else e.stdout
-            if stream:
-                _logger.error("git call error: %s", stream)
-            raise
-
-    def stdout(self, flag=True):
-        if flag is True:
-            return self.with_config(stdout=subprocess.PIPE)
-        elif flag is False:
-            return self.with_config(stdout=None)
-        return self.with_config(stdout=flag)
-
-    def lazy(self):
-        r = self.with_config()
-        r._config.pop('check', None)
-        r._opener = subprocess.Popen
-        return r
-
-    def check(self, flag):
-        return self.with_config(check=flag)
-
-    def with_config(self, **kw):
-        opts = {**self._config, **kw}
-        r = Repo(self._directory, **opts)
-        r._opener = self._opener
-        r._params = self._params
-        return r
-
-    def with_params(self, *args):
-        r = self.with_config()
-        r._params = args
-        return r
-
-    def clone(self, to, branch=None):
-        self._run(
-            'clone',
-            *([] if branch is None else ['-b', branch]),
-            self._directory, to,
-        )
-        return Repo(to)
-
-class GitCommand:
-    def __init__(self, repo, name):
-        self._name = name
-        self._repo = repo
-
-    def __call__(self, *args, **kwargs):
-        return self._repo._run(self._name, *args, *self._to_options(kwargs))
-
-    def _to_options(self, d):
-        for k, v in d.items():
-            if len(k) == 1:
-                yield '-' + k
-            else:
-                yield '--' + k.replace('_', '-')
-            if v not in (None, True):
-                assert v is not False
-                yield str(v)
 
 class CherrypickError(Exception):
     ...
diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py
index 04c23c7c..1f6ae7f4 100644
--- a/runbot_merge/__manifest__.py
+++ b/runbot_merge/__manifest__.py
@@ -7,6 +7,7 @@
         'security/ir.model.access.csv',
 
         'data/merge_cron.xml',
+        'models/crons/git_maintenance.xml',
         'data/runbot_merge.pull_requests.feedback.template.csv',
         'views/res_partner.xml',
         'views/runbot_merge_project.xml',
diff --git a/runbot_merge/git.py b/runbot_merge/git.py
new file mode 100644
index 00000000..91297f9e
--- /dev/null
+++ b/runbot_merge/git.py
@@ -0,0 +1,124 @@
+import dataclasses
+import itertools
+import logging
+import pathlib
+import resource
+import subprocess
+from typing import Optional, TypeVar
+
+from odoo.tools.appdirs import user_cache_dir
+
+_logger = logging.getLogger(__name__)
+
+
+def source_url(repository, prefix: str) -> str:
+    return 'https://{}@github.com/{}'.format(
+        repository.project_id[f'{prefix}_token'],
+        repository.name,
+    )
+
+
+def get_local(repository, prefix: Optional[str]) -> 'Optional[Repo]':
+    repos_dir = pathlib.Path(user_cache_dir('mergebot'))
+    repos_dir.mkdir(parents=True, exist_ok=True)
+    # NB: `repository.name` is `$org/$name` so this will be a subdirectory, probably
+    repo_dir = repos_dir / repository.name
+
+    if repo_dir.is_dir():
+        return git(repo_dir)
+    elif prefix:
+        _logger.info("Cloning out %s to %s", repository.name, repo_dir)
+        subprocess.run(['git', 'clone', '--bare', source_url(repository, prefix), str(repo_dir)], check=True)
+        # bare repos don't have fetch specs by default, and fetching *into*
+        # them is a pain in the ass, configure fetch specs so `git fetch`
+        # works properly
+        repo = git(repo_dir)
+        repo.config('--add', 'remote.origin.fetch', '+refs/heads/*:refs/heads/*')
+        # negative refspecs require git 2.29
+        repo.config('--add', 'remote.origin.fetch', '^refs/heads/tmp.*')
+        repo.config('--add', 'remote.origin.fetch', '^refs/heads/staging.*')
+        return repo
+
+
+ALWAYS = ('gc.auto=0', 'maintenance.auto=0')
+
+
+def _bypass_limits():
+    resource.setrlimit(resource.RLIMIT_AS, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
+
+
+def git(directory: str) -> 'Repo':
+    return Repo(directory, check=True)
+
+
+Self = TypeVar("Self", bound="Repo")
+class Repo:
+    def __init__(self, directory, **config) -> None:
+        self._directory = str(directory)
+        config.setdefault('stderr', subprocess.PIPE)
+        self._config = config
+        self._params = ()
+
+    def __getattr__(self, name: str) -> 'GitCommand':
+        return GitCommand(self, name.replace('_', '-'))
+
+    def _run(self, *args, **kwargs) -> subprocess.CompletedProcess:
+        opts = {**self._config, **kwargs}
+        args = ('git', '-C', self._directory)\
+            + tuple(itertools.chain.from_iterable(('-c', p) for p in self._params + ALWAYS))\
+            + args
+        try:
+            return subprocess.run(args, preexec_fn=_bypass_limits, **opts)
+        except subprocess.CalledProcessError as e:
+            stream = e.stderr or e.stdout
+            if stream:
+                _logger.error("git call error: %s", stream)
+            raise
+
+    def stdout(self, flag: bool = True) -> Self:
+        if flag is True:
+            return self.with_config(stdout=subprocess.PIPE)
+        elif flag is False:
+            return self.with_config(stdout=None)
+        return self.with_config(stdout=flag)
+
+    def check(self, flag: bool) -> Self:
+        return self.with_config(check=flag)
+
+    def with_config(self, **kw) -> Self:
+        opts = {**self._config, **kw}
+        r = Repo(self._directory, **opts)
+        r._params = self._params
+        return r
+
+    def with_params(self, *args) -> Self:
+        r = self.with_config()
+        r._params = args
+        return r
+
+    def clone(self, to: str, branch: Optional[str] = None) -> Self:
+        self._run(
+            'clone',
+            *([] if branch is None else ['-b', branch]),
+            self._directory, to,
+        )
+        return Repo(to)
+
+
+@dataclasses.dataclass
+class GitCommand:
+    repo: Repo
+    name: str
+
+    def __call__(self, *args, **kwargs) -> subprocess.CompletedProcess:
+        return self.repo._run(self.name, *args, *self._to_options(kwargs))
+
+    def _to_options(self, d):
+        for k, v in d.items():
+            if len(k) == 1:
+                yield '-' + k
+            else:
+                yield '--' + k.replace('_', '-')
+            if v not in (None, True):
+                assert v is not False
+                yield str(v)
diff --git a/runbot_merge/models/__init__.py b/runbot_merge/models/__init__.py
index 5cf276c8..9ca405e4 100644
--- a/runbot_merge/models/__init__.py
+++ b/runbot_merge/models/__init__.py
@@ -4,3 +4,4 @@ from . import project
 from . import pull_requests
 from . import project_freeze
 from . import staging_cancel
+from . import crons
diff --git a/runbot_merge/models/crons/__init__.py b/runbot_merge/models/crons/__init__.py
new file mode 100644
index 00000000..939a44cc
--- /dev/null
+++ b/runbot_merge/models/crons/__init__.py
@@ -0,0 +1 @@
+from . import git_maintenance
diff --git a/runbot_merge/models/crons/git_maintenance.py b/runbot_merge/models/crons/git_maintenance.py
new file mode 100644
index 00000000..33262d7e
--- /dev/null
+++ b/runbot_merge/models/crons/git_maintenance.py
@@ -0,0 +1,37 @@
+import logging
+import subprocess
+
+from odoo import models
+from ...git import get_local
+
+
+_gc = logging.getLogger(__name__)
+class GC(models.TransientModel):
+    _name = 'runbot_merge.maintenance'
+    _description = "Weekly maintenance of... cache repos?"
+
+    def _run(self):
+        # lock out crons which use the local repo cache to avoid concurrency
+        # issues while we're GC-ing it
+        Stagings = self.env['runbot_merge.stagings']
+        crons = self.env.ref('runbot_merge.staging_cron', Stagings) | self.env.ref('forwardport.port_forward', Stagings)
+        if crons:
+            self.env.cr.execute("""
+                SELECT 1 FROM ir_cron
+                WHERE id = any(%s)
+                FOR UPDATE
+            """, [crons.ids])
+
+        # run on all repos with a forwardport target (~ forwardport enabled)
+        for repo in self.env['runbot_merge.repository'].search([]):
+            repo_git = get_local(repo, prefix=None)
+            if not repo:
+                continue
+
+            _gc.info('Running maintenance on %s', repo.name)
+            r = repo_git\
+                .stdout(True)\
+                .with_config(stderr=subprocess.STDOUT, text=True, check=False)\
+                .gc(aggressive=True, prune='now')
+            if r.returncode:
+                _gc.warning("Maintenance failure (status=%d):\n%s", r.returncode, r.stdout)
diff --git a/runbot_merge/models/crons/git_maintenance.xml b/runbot_merge/models/crons/git_maintenance.xml
new file mode 100644
index 00000000..6190834a
--- /dev/null
+++ b/runbot_merge/models/crons/git_maintenance.xml
@@ -0,0 +1,26 @@
+<odoo>
+    <record id="access_forwardport_maintenance" model="ir.model.access">
+        <field name="name">Access to maintenance is useless</field>
+        <field name="model_id" ref="model_runbot_merge_maintenance"/>
+        <field name="perm_read">0</field>
+        <field name="perm_create">0</field>
+        <field name="perm_write">0</field>
+        <field name="perm_unlink">0</field>
+    </record>
+
+    <record model="ir.cron" id="maintenance">
+        <field name="name">Maintenance of repo cache</field>
+        <field name="model_id" ref="model_runbot_merge_maintenance"/>
+        <field name="state">code</field>
+        <field name="code">model._run()</field>
+        <!--
+        run sunday morning as it can take a while, unlikely someone will need
+        to stage or forward-port stuff at that point
+        -->
+        <field name="nextcall" eval="datetime.utcnow() + relativedelta(weekday=6, hour=2, minute=0, second=0, microsecond=0)"/>
+        <field name="interval_number">1</field>
+        <field name="interval_type">weeks</field>
+        <field name="numbercall">-1</field>
+        <field name="doall" eval="False"/>
+    </record>
+</odoo>

From 86a1b5523e91f1c2a1e08d781f73b8906fefaa38 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 16 Aug 2023 15:31:42 +0200
Subject: [PATCH 036/192] [MOV] runbot_merge: all the staging creation code to
 a separate module

Move *almost* all the staging code to free functions, in a separate
module, and extensively typed.

The only bits which didn't move are:

- the entry point (the cron hook), because it has to be a model method
  in order to be called
- the `_build_merge_message` method, because it needs to be
  overridable

There's also a bit of an import mess, because the cron &
`_build_merge_message` need to call into the new module, but the new
module wants the types they belong to, so it's a bit circular.
---
 forwardport/models/project.py          |   3 +-
 runbot_merge/github.py                 |  45 +-
 runbot_merge/models/__init__.py        |   1 +
 runbot_merge/models/project.py         |   7 +-
 runbot_merge/models/pull_requests.py   | 676 +----------------------
 runbot_merge/models/stagings_create.py | 727 +++++++++++++++++++++++++
 runbot_merge/tests/test_multirepo.py   |   8 -
 7 files changed, 791 insertions(+), 676 deletions(-)
 create mode 100644 runbot_merge/models/stagings_create.py

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 573d0086..b71a08f5 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -37,6 +37,7 @@ from odoo.tools.sql import reverse_order
 from odoo.tools.appdirs import user_cache_dir
 from odoo.addons.runbot_merge import git, utils
 from odoo.addons.runbot_merge.models.pull_requests import RPLUS
+from odoo.addons.runbot_merge.models.stagings_create import Message
 
 footer = '\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n'
 
@@ -1064,7 +1065,7 @@ stderr:
 
     def _make_fp_message(self, commit):
         cmap = json.loads(self.commits_map)
-        msg = self._parse_commit_message(commit['commit']['message'])
+        msg = Message.from_message(commit['commit']['message'])
         # write the *merged* commit as "original", not the PR's
         msg.headers['x-original-commit'] = cmap.get(commit['sha'], commit['sha'])
         # don't stringify so caller can still perform alterations
diff --git a/runbot_merge/github.py b/runbot_merge/github.py
index ed3950b0..3fdec209 100644
--- a/runbot_merge/github.py
+++ b/runbot_merge/github.py
@@ -8,6 +8,7 @@ import pathlib
 import pprint
 import time
 import unicodedata
+from typing import Iterable, List, TypedDict, Literal
 
 import requests
 import werkzeug.urls
@@ -47,6 +48,42 @@ def _init_gh_logger():
 if odoo.netsvc._logger_init:
     _init_gh_logger()
 
+SimpleUser = TypedDict('SimpleUser', {
+    'login': str,
+    'url': str,
+    'type': Literal['User', 'Organization'],
+})
+Authorship = TypedDict('Authorship', {
+    'name': str,
+    'email': str,
+})
+Commit = TypedDict('Commit', {
+    'tree': str,
+    'url': str,
+    'message': str,
+    # optional when creating a commit
+    'author': Authorship,
+    'committer': Authorship,
+    'comments_count': int,
+})
+CommitLink = TypedDict('CommitLink', {
+    'html_url': str,
+    'sha': str,
+    'url': str,
+})
+PrCommit = TypedDict('PrCommit', {
+    'url': str,
+    'sha': str,
+    'commit': Commit,
+    # optional when creating a commit (in which case it uses the current user)
+    'author': SimpleUser,
+    'committer': SimpleUser,
+    'parents': List[CommitLink],
+    # not actually true but we're smuggling stuff via that key
+    'new_tree': str,
+})
+
+
 GH_LOG_PATTERN = """=> {method} {path}{qs}{body}
 
 <= {r.status_code} {r.reason}
@@ -137,7 +174,7 @@ class GH(object):
         r.raise_for_status()
         return r.json()
 
-    def head(self, branch):
+    def head(self, branch: str) -> str:
         d = utils.backoff(
             lambda: self('get', 'git/refs/heads/{}'.format(branch)).json(),
             exc=requests.HTTPError
@@ -383,14 +420,14 @@ class GH(object):
             if not r.links.get('next'):
                 return
 
-    def commits_lazy(self, pr):
+    def commits_lazy(self, pr: int) -> Iterable[PrCommit]:
         for page in itertools.count(1):
-            r = self('get', 'pulls/{}/commits'.format(pr), params={'page': page})
+            r = self('get', f'pulls/{pr}/commits', params={'page': page})
             yield from r.json()
             if not r.links.get('next'):
                 return
 
-    def commits(self, pr):
+    def commits(self, pr: int) -> List[PrCommit]:
         """ Returns a PR's commits oldest first (that's what GH does &
         is what we want)
         """
diff --git a/runbot_merge/models/__init__.py b/runbot_merge/models/__init__.py
index 9ca405e4..6cbd92cf 100644
--- a/runbot_merge/models/__init__.py
+++ b/runbot_merge/models/__init__.py
@@ -3,5 +3,6 @@ from . import res_partner
 from . import project
 from . import pull_requests
 from . import project_freeze
+from . import stagings_create
 from . import staging_cancel
 from . import crons
diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index 8578aeb5..67222ea8 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -32,8 +32,9 @@ class Project(models.Model):
         required=True,
         default="hanson", # mergebot du bot du bot du~
         help="Prefix (~bot name) used when sending commands from PR "
-             "comments e.g. [hanson retry] or [hanson r+ p=1]"
+             "comments e.g. [hanson retry] or [hanson r+ p=1]",
     )
+    github_name = fields.Char(related='github_prefix')
 
     batch_limit = fields.Integer(
         default=8, group_operator=None, help="Maximum number of PRs staged together")
@@ -64,6 +65,8 @@ class Project(models.Model):
                     self.env.cr.commit()
 
     def _create_stagings(self, commit=False):
+        from .stagings_create import try_staging
+
         # look up branches which can be staged on and have no active staging
         for branch in self.env['runbot_merge.branch'].search([
             ('active_staging_id', '=', False),
@@ -74,7 +77,7 @@ class Project(models.Model):
                 with self.env.cr.savepoint(), \
                     sentry_sdk.start_span(description=f'create staging {branch.name}') as span:
                     span.set_tag('branch', branch.name)
-                    branch.try_staging()
+                    try_staging(branch)
             except Exception:
                 _logger.exception("Failed to create staging for branch %r", branch.name)
             else:
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 0005704b..a392b880 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1,37 +1,24 @@
-# coding: utf-8
-
 import ast
-import base64
 import collections
 import contextlib
 import datetime
-import io
 import itertools
 import json
 import logging
-import os
 import pprint
 import re
 import time
+from typing import Optional, Union
 
-from difflib import Differ
-from itertools import takewhile
-from typing import Optional
-
-import requests
 import sentry_sdk
 import werkzeug
-from werkzeug.datastructures import Headers
 
 from odoo import api, fields, models, tools
 from odoo.exceptions import ValidationError
 from odoo.osv import expression
-from odoo.tools import OrderedSet
 
 from .. import github, exceptions, controllers, utils
 
-WAIT_FOR_VISIBILITY = [10, 10, 10, 10]
-
 _logger = logging.getLogger(__name__)
 
 
@@ -67,6 +54,8 @@ class Repository(models.Model):
     _name = _description = 'runbot_merge.repository'
     _order = 'sequence, id'
 
+    id: int
+
     sequence = fields.Integer(default=50, group_operator=None)
     name = fields.Char(required=True)
     project_id = fields.Many2one('runbot_merge.project', required=True, index=True)
@@ -98,7 +87,7 @@ All substitutions are tentatively applied sequentially to the input.
             vals['status_ids'] = [(5, 0, {})] + [(0, 0, {'context': c}) for c in st.split(',')]
         return super().write(vals)
 
-    def github(self, token_field='github_token'):
+    def github(self, token_field='github_token') -> github.GH:
         return github.GH(self.project_id[token_field], self.name)
 
     def _auto_init(self):
@@ -245,6 +234,8 @@ class Branch(models.Model):
     _name = _description = 'runbot_merge.branch'
     _order = 'sequence, name'
 
+    id: int
+
     name = fields.Char(required=True)
     project_id = fields.Many2one('runbot_merge.project', required=True, index=True)
 
@@ -298,235 +289,6 @@ class Branch(models.Model):
         for b in self:
             b.active_staging_id = b.with_context(active_test=True).staging_ids
 
-    def _ready(self):
-        self.env.cr.execute("""
-        SELECT
-          min(pr.priority) as priority,
-          array_agg(pr.id) AS match
-        FROM runbot_merge_pull_requests pr
-        WHERE pr.target = any(%s)
-          -- exclude terminal states (so there's no issue when
-          -- deleting branches & reusing labels)
-          AND pr.state != 'merged'
-          AND pr.state != 'closed'
-        GROUP BY
-            pr.target,
-            CASE
-                WHEN pr.label SIMILAR TO '%%:patch-[[:digit:]]+'
-                    THEN pr.id::text
-                ELSE pr.label
-            END
-        HAVING
-            bool_or(pr.state = 'ready') or bool_or(pr.priority = 0)
-        ORDER BY min(pr.priority), min(pr.id)
-        """, [self.ids])
-        browse = self.env['runbot_merge.pull_requests'].browse
-        return [(p, browse(ids)) for p, ids in self.env.cr.fetchall()]
-
-    def _stageable(self):
-        return [
-            (p, prs)
-            for p, prs in self._ready()
-            if not any(prs.mapped('blocked'))
-        ]
-
-    def try_staging(self):
-        """ Tries to create a staging if the current branch does not already
-        have one. Returns None if the branch already has a staging or there
-        is nothing to stage, the newly created staging otherwise.
-        """
-        logger = _logger.getChild('cron')
-
-        logger.info(
-            "Checking %s (%s) for staging: %s, skip? %s",
-            self, self.name,
-            self.active_staging_id,
-            bool(self.active_staging_id)
-        )
-        if self.active_staging_id:
-            return
-
-        rows = self._stageable()
-        priority = rows[0][0] if rows else -1
-        if priority == 0 or priority == 1:
-            # p=0 take precedence over all else
-            # p=1 allows merging a fix inside / ahead of a split (e.g. branch
-            # is broken or widespread false positive) without having to cancel
-            # the existing staging
-            batched_prs = [pr_ids for _, pr_ids in takewhile(lambda r: r[0] == priority, rows)]
-        elif self.split_ids:
-            split_ids = self.split_ids[0]
-            logger.info("Found split of PRs %s, re-staging", split_ids.mapped('batch_ids.prs'))
-            batched_prs = [batch.prs for batch in split_ids.batch_ids]
-            split_ids.unlink()
-        else: # p=2
-            batched_prs = [pr_ids for _, pr_ids in takewhile(lambda r: r[0] == priority, rows)]
-
-        if not batched_prs:
-            return
-
-        Batch = self.env['runbot_merge.batch']
-        staged = Batch
-        original_heads = {}
-        meta = {repo: {} for repo in self.project_id.repo_ids.having_branch(self)}
-        for repo, it in meta.items():
-            gh = it['gh'] = repo.github()
-            it['head'] = original_heads[repo] = gh.head(self.name)
-            # create tmp staging branch
-            gh.set_ref('tmp.{}'.format(self.name), it['head'])
-
-        batch_limit = self.project_id.batch_limit
-        first = True
-        for batch in batched_prs:
-            if len(staged) >= batch_limit:
-                break
-            try:
-                staged |= Batch.stage(meta, batch)
-            except exceptions.MergeError as e:
-                pr = e.args[0]
-                _logger.exception("Failed to merge %s into staging branch", pr.display_name)
-                if first or isinstance(e, exceptions.Unmergeable):
-                    if len(e.args) > 1 and e.args[1]:
-                        reason = e.args[1]
-                    else:
-                        reason = e.__cause__ or e.__context__
-                    # if the reason is a json document, assume it's a github
-                    # error and try to extract the error message to give it to
-                    # the user
-                    with contextlib.suppress(Exception):
-                        reason = json.loads(str(reason))['message'].lower()
-
-                    pr.state = 'error'
-                    self.env.ref('runbot_merge.pr.merge.failed')._send(
-                        repository=pr.repository,
-                        pull_request=pr.number,
-                        format_args= {'pr': pr, 'reason': reason, 'exc': e},
-                    )
-            else:
-                first = False
-
-        if not staged:
-            return
-
-        heads = []
-        heads_map = {}
-        commits = []
-        for repo, it in meta.items():
-            tree = it['gh'].commit(it['head'])['tree']
-            # ensures staging branches are unique and always
-            # rebuilt
-            r = base64.b64encode(os.urandom(12)).decode('ascii')
-            trailer = ''
-            if heads_map:
-                trailer = '\n'.join(
-                    'Runbot-dependency: %s:%s' % (repo, h)
-                    for repo, h in heads_map.items()
-                )
-            dummy_head = {'sha': it['head']}
-            if it['head'] == original_heads[repo]:
-                # if the repo has not been updated by the staging, create a
-                # dummy commit to force rebuild
-                dummy_head = it['gh']('post', 'git/commits', json={
-                    'message': '''force rebuild
-
-uniquifier: %s
-For-Commit-Id: %s
-%s''' % (r, it['head'], trailer),
-                    'tree': tree['sha'],
-                    'parents': [it['head']],
-                }).json()
-
-            # special case if the two commits are identical because otherwise
-            # postgres raises error "ensure that no rows proposed for insertion
-            # within the same command have duplicate constained values"
-            if it['head'] == dummy_head['sha']:
-                self.env.cr.execute(
-                    "INSERT INTO runbot_merge_commit (sha, to_check, statuses) "
-                    "VALUES (%s, true, '{}') "
-                    "ON CONFLICT (sha) DO UPDATE SET to_check=true "
-                    "RETURNING id",
-                    [it['head']]
-                )
-                [commit] = [head] = self.env.cr.fetchone()
-            else:
-                self.env.cr.execute(
-                    "INSERT INTO runbot_merge_commit (sha, to_check, statuses) "
-                    "VALUES (%s, false, '{}'), (%s, true, '{}') "
-                    "ON CONFLICT (sha) DO UPDATE SET to_check=true "
-                    "RETURNING id",
-                    [it['head'], dummy_head['sha']]
-                )
-                ([commit], [head]) = self.env.cr.fetchall()
-
-            heads_map[repo.name] = dummy_head['sha']
-            heads.append(fields.Command.create({
-                'repository_id': repo.id,
-                'commit_id': head,
-            }))
-            commits.append(fields.Command.create({
-                'repository_id': repo.id,
-                'commit_id': commit,
-            }))
-
-        # create actual staging object
-        st = self.env['runbot_merge.stagings'].create({
-            'target': self.id,
-            'batch_ids': [(4, batch.id, 0) for batch in staged],
-            'heads': heads,
-            'commits': commits,
-        })
-        # create staging branch from tmp
-        token = self.project_id.github_token
-        for r in self.project_id.repo_ids.having_branch(self):
-            it = meta[r]
-            staging_head = heads_map[r.name]
-            _logger.info(
-                "%s: create staging for %s:%s at %s",
-                self.project_id.name, r.name, self.name,
-                staging_head
-            )
-            refname = 'staging.{}'.format(self.name)
-            it['gh'].set_ref(refname, staging_head)
-
-            i = itertools.count()
-            @utils.backoff(delays=WAIT_FOR_VISIBILITY, exc=TimeoutError)
-            def wait_for_visibility():
-                if self._check_visibility(r, refname, staging_head, token):
-                    _logger.info(
-                        "[repo] updated %s:%s to %s: ok (at %d/%d)",
-                        r.name, refname, staging_head,
-                        next(i), len(WAIT_FOR_VISIBILITY)
-                    )
-                    return
-                _logger.warning(
-                    "[repo] updated %s:%s to %s: failed (at %d/%d)",
-                    r.name, refname, staging_head,
-                    next(i), len(WAIT_FOR_VISIBILITY)
-                )
-                raise TimeoutError("Staged head not updated after %d seconds" % sum(WAIT_FOR_VISIBILITY))
-
-        logger.info("Created staging %s (%s) to %s", st, ', '.join(
-            '%s[%s]' % (batch, batch.prs)
-            for batch in staged
-        ), st.target.name)
-        return st
-
-    def _check_visibility(self, repo, branch_name, expected_head, token):
-        """ Checks the repository actual to see if the new / expected head is
-        now visible
-        """
-        # v1 protocol provides URL for ref discovery: https://github.com/git/git/blob/6e0cc6776106079ed4efa0cc9abace4107657abf/Documentation/technical/http-protocol.txt#L187
-        # for more complete client this is also the capabilities discovery and
-        # the "entry point" for the service
-        url = 'https://github.com/{}.git/info/refs?service=git-upload-pack'.format(repo.name)
-        with requests.get(url, stream=True, auth=(token, '')) as resp:
-            if not resp.ok:
-                return False
-            for head, ref in parse_refs_smart(resp.raw.read):
-                if ref != ('refs/heads/' + branch_name):
-                    continue
-                return head == expected_head
-            return False
 
 ACL = collections.namedtuple('ACL', 'is_admin is_reviewer is_author')
 class PullRequests(models.Model):
@@ -534,6 +296,9 @@ class PullRequests(models.Model):
     _order = 'number desc'
     _rec_name = 'number'
 
+    id: int
+    display_name: str
+
     target = fields.Many2one('runbot_merge.branch', required=True, index=True)
     repository = fields.Many2one('runbot_merge.repository', required=True)
     # NB: check that target & repo have same project & provide project related?
@@ -1265,35 +1030,14 @@ class PullRequests(models.Model):
             if commit:
                 self.env.cr.commit()
 
-    def _parse_commit_message(self, message):
-        """ Parses a commit message to split out the pseudo-headers (which
-        should be at the end) from the body, and serialises back with a
-        predefined pseudo-headers ordering.
-        """
-        return Message.from_message(message)
-
-    def _is_mentioned(self, message, *, full_reference=False):
-        """Returns whether ``self`` is mentioned in ``message```
-
-        :param str | PullRequest message:
-        :param bool full_reference: whether the repository name must be present
-        :rtype: bool
-        """
-        if full_reference:
-            pattern = fr'\b{re.escape(self.display_name)}\b'
-        else:
-            repository = self.repository.name # .replace('/', '\\/')
-            pattern = fr'( |\b{repository})#{self.number}\b'
-        return bool(re.search(pattern, message if isinstance(message, str) else message.message))
-
-    def _build_merge_message(self, message, related_prs=()):
+    def _build_merge_message(self, message: Union['PullRequests', str], related_prs=()) -> 'Message':
         # handle co-authored commits (https://help.github.com/articles/creating-a-commit-with-multiple-authors/)
-        m = self._parse_commit_message(message)
-        if not self._is_mentioned(message):
-            m.body += '\n\ncloses {pr.display_name}'.format(pr=self)
+        m = Message.from_message(message)
+        if not is_mentioned(message, self):
+            m.body += f'\n\ncloses {self.display_name}'
 
         for r in related_prs:
-            if not r._is_mentioned(message, full_reference=True):
+            if not is_mentioned(message, r, full_reference=True):
                 m.headers.add('Related', r.display_name)
 
         if self.reviewed_by:
@@ -1301,190 +1045,6 @@ class PullRequests(models.Model):
 
         return m
 
-    def _add_self_references(self, commits):
-        """Adds a footer reference to ``self`` to all ``commits`` if they don't
-        already refer to the PR.
-        """
-        for c in (c['commit'] for c in commits):
-            if not self._is_mentioned(c['message']):
-                m = self._parse_commit_message(c['message'])
-                m.headers.pop('Part-Of', None)
-                m.headers.add('Part-Of', self.display_name)
-                c['message'] = str(m)
-
-    def _stage(self, gh, target, related_prs=()):
-        # nb: pr_commits is oldest to newest so pr.head is pr_commits[-1]
-        _, prdict = gh.pr(self.number)
-        commits = prdict['commits']
-        method = self.merge_method or ('rebase-ff' if commits == 1 else None)
-        if commits > 50 and method.startswith('rebase'):
-            raise exceptions.Unmergeable(self, "Rebasing 50 commits is too much.")
-        if commits > 250:
-            raise exceptions.Unmergeable(
-                self, "Merging PRs of 250 or more commits is not supported "
-                "(https://developer.github.com/v3/pulls/#list-commits-on-a-pull-request)"
-            )
-        pr_commits = gh.commits(self.number)
-        for c in pr_commits:
-            if not (c['commit']['author']['email'] and c['commit']['committer']['email']):
-                raise exceptions.Unmergeable(
-                    self,
-                    f"All commits must have author and committer email, "
-                    f"missing email on {c['sha']} indicates the authorship is "
-                    f"most likely incorrect."
-                )
-
-        # sync and signal possibly missed updates
-        invalid = {}
-        diff = []
-        pr_head = pr_commits[-1]['sha']
-        if self.head != pr_head:
-            invalid['head'] = pr_head
-            diff.append(('Head', self.head, pr_head))
-
-        if self.target.name != prdict['base']['ref']:
-            branch = self.env['runbot_merge.branch'].with_context(active_test=False).search([
-                ('name', '=', prdict['base']['ref']),
-                ('project_id', '=', self.repository.project_id.id),
-            ])
-            if not branch:
-                self.unlink()
-                raise exceptions.Unmergeable(self, "While staging, found this PR had been retargeted to an un-managed branch.")
-            invalid['target'] = branch.id
-            diff.append(('Target branch', self.target.name, branch.name))
-
-        if self.squash != commits == 1:
-            invalid['squash'] = commits == 1
-            diff.append(('Single commit', self.squash, commits == 1))
-
-        msg = utils.make_message(prdict)
-        if self.message != msg:
-            invalid['message'] = msg
-            diff.append(('Message', self.message, msg))
-
-        if invalid:
-            self.write({**invalid, 'state': 'opened', 'head': pr_head})
-            raise exceptions.Mismatch(invalid, diff)
-
-        if self.reviewed_by and self.reviewed_by.name == self.reviewed_by.github_login:
-            # XXX: find other trigger(s) to sync github name?
-            gh_name = gh.user(self.reviewed_by.github_login)['name']
-            if gh_name:
-                self.reviewed_by.name = gh_name
-
-        # NOTE: lost merge v merge/copy distinction (head being
-        #       a merge commit reused instead of being re-merged)
-        return method, getattr(self, '_stage_' + method.replace('-', '_'))(
-            gh, target, pr_commits, related_prs=related_prs)
-
-    def _stage_squash(self, gh, target, commits, related_prs=()):
-        msg = self._build_merge_message(self, related_prs=related_prs)
-        authorship = {}
-
-        authors = {
-            (c['commit']['author']['name'], c['commit']['author']['email'])
-            for c in commits
-        }
-        if len(authors) == 1:
-            name, email = authors.pop()
-            authorship['author']  = {'name': name, 'email': email}
-        else:
-            msg.headers.extend(sorted(
-                ('Co-Authored-By', "%s <%s>" % author)
-                for author in authors
-            ))
-
-        committers = {
-            (c['commit']['committer']['name'], c['commit']['committer']['email'])
-            for c in commits
-        }
-        if len(committers) == 1:
-            name, email = committers.pop()
-            authorship['committer'] = {'name': name, 'email': email}
-        # should committers also be added to co-authors?
-
-        original_head = gh.head(target)
-        merge_tree = gh.merge(self.head, target, 'temp merge')['tree']['sha']
-        head = gh('post', 'git/commits', json={
-            **authorship,
-            'message': str(msg),
-            'tree': merge_tree,
-            'parents': [original_head],
-        }).json()['sha']
-        gh.set_ref(target, head)
-
-        commits_map = {c['sha']: head for c in commits}
-        commits_map[''] = head
-        self.commits_map = json.dumps(commits_map)
-
-        return head
-
-    def _stage_rebase_ff(self, gh, target, commits, related_prs=()):
-        # updates head commit with PR number (if necessary) then rebases
-        # on top of target
-        msg = self._build_merge_message(commits[-1]['commit']['message'], related_prs=related_prs)
-        commits[-1]['commit']['message'] = str(msg)
-        self._add_self_references(commits[:-1])
-        head, mapping = gh.rebase(self.number, target, commits=commits)
-        self.commits_map = json.dumps({**mapping, '': head})
-        return head
-
-    def _stage_rebase_merge(self, gh, target, commits, related_prs=()):
-        self._add_self_references(commits)
-        h, mapping = gh.rebase(self.number, target, reset=True, commits=commits)
-        msg = self._build_merge_message(self, related_prs=related_prs)
-        merge_head = gh.merge(h, target, str(msg))['sha']
-        self.commits_map = json.dumps({**mapping, '': merge_head})
-        return merge_head
-
-    def _stage_merge(self, gh, target, commits, related_prs=()):
-        pr_head = commits[-1] # oldest to newest
-        base_commit = None
-        head_parents = {p['sha'] for p in pr_head['parents']}
-        if len(head_parents) > 1:
-            # look for parent(s?) of pr_head not in PR, means it's
-            # from target (so we merged target in pr)
-            merge = head_parents - {c['sha'] for c in commits}
-            external_parents = len(merge)
-            if external_parents > 1:
-                raise exceptions.Unmergeable(
-                    "The PR head can only have one parent from the base branch "
-                    "(not part of the PR itself), found %d: %s" % (
-                        external_parents,
-                        ', '.join(merge)
-                    ))
-            if external_parents == 1:
-                [base_commit] = merge
-
-        commits_map = {c['sha']: c['sha'] for c in commits}
-        if base_commit:
-            # replicate pr_head with base_commit replaced by
-            # the current head
-            original_head = gh.head(target)
-            merge_tree = gh.merge(pr_head['sha'], target, 'temp merge')['tree']['sha']
-            new_parents = [original_head] + list(head_parents - {base_commit})
-            msg = self._build_merge_message(pr_head['commit']['message'], related_prs=related_prs)
-            copy = gh('post', 'git/commits', json={
-                'message': str(msg),
-                'tree': merge_tree,
-                'author': pr_head['commit']['author'],
-                'committer': pr_head['commit']['committer'],
-                'parents': new_parents,
-            }).json()
-            gh.set_ref(target, copy['sha'])
-            # merge commit *and old PR head* map to the pr head replica
-            commits_map[''] = commits_map[pr_head['sha']] = copy['sha']
-            self.commits_map = json.dumps(commits_map)
-            return copy['sha']
-        else:
-            # otherwise do a regular merge
-            msg = self._build_merge_message(self)
-            merge_head = gh.merge(self.head, target, str(msg))['sha']
-            # and the merge commit is the normal merge head
-            commits_map[''] = merge_head
-            self.commits_map = json.dumps(commits_map)
-            return merge_head
-
     def unstage(self, reason, *args):
         """ If the PR is staged, cancel the staging. If the PR is split and
         waiting, remove it from the split (possibly delete the split entirely)
@@ -2241,82 +1801,6 @@ class Batch(models.Model):
                     raise ValidationError("All prs of a batch must have different target repositories, got a duplicate %s on %s" % (pr.repository, pr))
                 repos |= pr.repository
 
-    def stage(self, meta, prs):
-        """
-        Updates meta[*][head] on success
-
-        :return: () or Batch object (if all prs successfully staged)
-        """
-        new_heads = {}
-        pr_fields = self.env['runbot_merge.pull_requests']._fields
-        for pr in prs:
-            gh = meta[pr.repository]['gh']
-
-            _logger.info(
-                "Staging pr %s for target %s; method=%s",
-                pr.display_name, pr.target.name,
-                pr.merge_method or (pr.squash and 'single') or None
-            )
-
-            target = 'tmp.{}'.format(pr.target.name)
-            original_head = gh.head(target)
-            try:
-                try:
-                    method, new_heads[pr] = pr._stage(gh, target, related_prs=(prs - pr))
-                    _logger.info(
-                        "Staged pr %s to %s by %s: %s -> %s",
-                        pr.display_name, pr.target.name, method,
-                        original_head, new_heads[pr]
-                    )
-                except Exception:
-                    # reset the head which failed, as rebase() may have partially
-                    # updated it (despite later steps failing)
-                    gh.set_ref(target, original_head)
-                    # then reset every previous update
-                    for to_revert in new_heads.keys():
-                        it = meta[to_revert.repository]
-                        it['gh'].set_ref('tmp.{}'.format(to_revert.target.name), it['head'])
-                    raise
-            except github.MergeError as e:
-                raise exceptions.MergeError(pr) from e
-            except exceptions.Mismatch as e:
-                def format_items(items):
-                    """ Bit of a pain in the ass because difflib really wants
-                    all lines to be newline-terminated, but not all values are
-                    actual lines, and also needs to split multiline values.
-                    """
-                    for name, value in items:
-                        yield name + ':\n'
-                        if not value.endswith('\n'):
-                            value += '\n'
-                        yield from value.splitlines(keepends=True)
-                        yield '\n'
-
-                old = list(format_items((n, str(v)) for n, v, _ in e.args[1]))
-                new = list(format_items((n, str(v)) for n, _, v in e.args[1]))
-                diff = ''.join(Differ().compare(old, new))
-                _logger.info("data mismatch on %s:\n%s", pr.display_name, diff)
-                self.env.ref('runbot_merge.pr.staging.mismatch')._send(
-                    repository=pr.repository,
-                    pull_request=pr.number,
-                    format_args={
-                        'pr': pr,
-                        'mismatch': ', '.join(pr_fields[f].string for f in e.args[0]),
-                        'diff': diff,
-                        'unchecked': ', '.join(pr_fields[f].string for f in UNCHECKABLE)
-                    }
-                )
-                return self.env['runbot_merge.batch']
-
-        # update meta to new heads
-        for pr, head in new_heads.items():
-            meta[pr.repository]['head'] = head
-        return self.create({
-            'target': prs[0].target.id,
-            'prs': [(4, pr.id, 0) for pr in prs],
-        })
-
-UNCHECKABLE = ['merge_method', 'overrides', 'draft']
 
 class FetchJob(models.Model):
     _name = _description = 'runbot_merge.fetch_job'
@@ -2372,134 +1856,4 @@ def to_status(v):
         return v
     return {'state': v, 'target_url': None, 'description': None}
 
-refline = re.compile(rb'([\da-f]{40}) ([^\0\n]+)(\0.*)?\n?$')
-ZERO_REF = b'0'*40
-def parse_refs_smart(read):
-    """ yields pkt-line data (bytes), or None for flush lines """
-    def read_line():
-        length = int(read(4), 16)
-        if length == 0:
-            return None
-        return read(length - 4)
-
-    header = read_line()
-    assert header.rstrip() == b'# service=git-upload-pack', header
-    assert read_line() is None, "failed to find first flush line"
-    # read lines until second delimiter
-    for line in iter(read_line, None):
-        if line.startswith(ZERO_REF):
-            break # empty list (no refs)
-        m = refline.match(line)
-        yield m[1].decode(), m[2].decode()
-
-BREAK = re.compile(r'''
-    ^
-    [ ]{0,3} # 0-3 spaces of indentation
-    # followed by a sequence of three or more matching -, _, or * characters,
-    # each followed optionally by any number of spaces or tabs
-    # so needs to start with a _, - or *, then have at least 2 more such
-    # interspersed with any number of spaces or tabs
-    ([*_-])
-    ([ \t]*\1){2,}
-    [ \t]*
-    $
-''', flags=re.VERBOSE)
-SETEX_UNDERLINE = re.compile(r'''
-    ^
-    [ ]{0,3} # no more than 3 spaces indentation
-    [-=]+ # a sequence of = characters or a sequence of - characters
-    [ ]* # any number of trailing spaces
-    $
-    # we don't care about "a line containing a single -" because we want to
-    # disambiguate SETEX headings from thematic breaks, and thematic breaks have
-    # 3+ -. Doesn't look like GH interprets `- - -` as a line so yay...
-''', flags=re.VERBOSE)
-HEADER = re.compile('^([A-Za-z-]+): (.*)$')
-class Message:
-    @classmethod
-    def from_message(cls, msg):
-        in_headers = True
-        maybe_setex = None
-        # creating from PR message -> remove content following break
-        msg, handle_break = (msg, False) if isinstance(msg, str) else (msg.message, True)
-        headers = []
-        body = []
-        # don't process the title (first line) of the commit message
-        msg = msg.splitlines()
-        for line in reversed(msg[1:]):
-            if maybe_setex:
-                # NOTE: actually slightly more complicated: it's a SETEX heading
-                #       only if preceding line(s) can be interpreted as a
-                #       paragraph so e.g. a title followed by a line of dashes
-                #       would indeed be a break, but this should be good enough
-                #       for now, if we need more we'll need a full-blown
-                #       markdown parser probably
-                if line: # actually a SETEX title -> add underline to body then process current
-                    body.append(maybe_setex)
-                else: # actually break, remove body then process current
-                    body = []
-                maybe_setex = None
-
-            if not line:
-                if not in_headers and body and body[-1]:
-                    body.append(line)
-                continue
-
-            if handle_break and BREAK.match(line):
-                if SETEX_UNDERLINE.match(line):
-                    maybe_setex = line
-                else:
-                    body = []
-                continue
-
-            h = HEADER.match(line)
-            if h:
-                # c-a-b = special case from an existing test, not sure if actually useful?
-                if in_headers or h.group(1).lower() == 'co-authored-by':
-                    headers.append(h.groups())
-                    continue
-
-            body.append(line)
-            in_headers = False
-
-        # if there are non-title body lines, add a separation after the title
-        if body and body[-1]:
-            body.append('')
-        body.append(msg[0])
-        return cls('\n'.join(reversed(body)), Headers(reversed(headers)))
-
-    def __init__(self, body, headers=None):
-        self.body = body
-        self.headers = headers or Headers()
-
-    def __setattr__(self, name, value):
-        # make sure stored body is always stripped
-        if name == 'body':
-            value = value and value.strip()
-        super().__setattr__(name, value)
-
-    def __str__(self):
-        if not self.headers:
-            return self.body + '\n'
-
-        with io.StringIO(self.body) as msg:
-            msg.write(self.body)
-            msg.write('\n\n')
-            # https://git.wiki.kernel.org/index.php/CommitMessageConventions
-            # seems to mostly use capitalised names (rather than title-cased)
-            keys = list(OrderedSet(k.capitalize() for k in self.headers.keys()))
-            # c-a-b must be at the very end otherwise github doesn't see it
-            keys.sort(key=lambda k: k == 'Co-authored-by')
-            for k in keys:
-                for v in self.headers.getlist(k):
-                    msg.write(k)
-                    msg.write(': ')
-                    msg.write(v)
-                    msg.write('\n')
-
-            return msg.getvalue()
-
-    def sub(self, pattern, repl, *, flags):
-        """ Performs in-place replacements on the body
-        """
-        self.body = re.sub(pattern, repl, self.body, flags=flags)
+from .stagings_create import is_mentioned, Message
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
new file mode 100644
index 00000000..b00342dc
--- /dev/null
+++ b/runbot_merge/models/stagings_create.py
@@ -0,0 +1,727 @@
+import base64
+import contextlib
+import dataclasses
+import io
+import json
+import logging
+import os
+import re
+import tempfile
+from difflib import Differ
+from itertools import count, takewhile
+from pathlib import Path
+from typing import Dict, Union, Optional, Literal, Callable, Iterator, Tuple, List, TypeAlias
+
+import requests
+from werkzeug.datastructures import Headers
+
+from odoo import api, models, fields
+from odoo.tools import OrderedSet
+from odoo.tools.appdirs import user_cache_dir
+from .pull_requests import Branch, Stagings, PullRequests, Repository, Batch
+from .. import exceptions, utils, github, git
+
+WAIT_FOR_VISIBILITY = [10, 10, 10, 10]
+_logger = logging.getLogger(__name__)
+
+
+class Project(models.Model):
+    _inherit = 'runbot_merge.project'
+
+
+@dataclasses.dataclass(slots=True)
+class StagingSlice:
+    """Staging state for a single repository:
+
+    - gh is a cache for the github proxy object (contains a session for reusing
+      connection)
+    - head is the current staging head for the branch of that repo
+    - working_copy is the local working copy for the staging for that repo
+    """
+    gh: github.GH
+    head: str
+    working_copy: git.Repo
+
+
+StagingState: TypeAlias = Dict[Repository, StagingSlice]
+
+def try_staging(branch: Branch) -> Optional[Stagings]:
+    """ Tries to create a staging if the current branch does not already
+    have one. Returns None if the branch already has a staging or there
+    is nothing to stage, the newly created staging otherwise.
+    """
+    _logger.info(
+        "Checking %s (%s) for staging: %s, skip? %s",
+        branch, branch.name,
+        branch.active_staging_id,
+        bool(branch.active_staging_id)
+    )
+    if branch.active_staging_id:
+        return None
+
+    rows = [
+        (p, prs)
+        for p, prs in ready_prs(for_branch=branch)
+        if not any(prs.mapped('blocked'))
+    ]
+    if not rows:
+        return
+
+    priority = rows[0][0]
+    if priority == 0 or priority == 1:
+        # p=0 take precedence over all else
+        # p=1 allows merging a fix inside / ahead of a split (e.g. branch
+        # is broken or widespread false positive) without having to cancel
+        # the existing staging
+        batched_prs = [pr_ids for _, pr_ids in takewhile(lambda r: r[0] == priority, rows)]
+    elif branch.split_ids:
+        split_ids = branch.split_ids[0]
+        _logger.info("Found split of PRs %s, re-staging", split_ids.mapped('batch_ids.prs'))
+        batched_prs = [batch.prs for batch in split_ids.batch_ids]
+        split_ids.unlink()
+    else: # p=2
+        batched_prs = [pr_ids for _, pr_ids in takewhile(lambda r: r[0] == priority, rows)]
+
+    with contextlib.ExitStack() as cleanup:
+        return stage_into(branch, batched_prs, cleanup)
+
+
+def ready_prs(for_branch: Branch) -> List[Tuple[int, PullRequests]]:
+    env = for_branch.env
+    env.cr.execute("""
+    SELECT
+      min(pr.priority) as priority,
+      array_agg(pr.id) AS match
+    FROM runbot_merge_pull_requests pr
+    WHERE pr.target = any(%s)
+      -- exclude terminal states (so there's no issue when
+      -- deleting branches & reusing labels)
+      AND pr.state != 'merged'
+      AND pr.state != 'closed'
+    GROUP BY
+        pr.target,
+        CASE
+            WHEN pr.label SIMILAR TO '%%:patch-[[:digit:]]+'
+                THEN pr.id::text
+            ELSE pr.label
+        END
+    HAVING
+        bool_or(pr.state = 'ready') or bool_or(pr.priority = 0)
+    ORDER BY min(pr.priority), min(pr.id)
+    """, [for_branch.ids])
+    browse = env['runbot_merge.pull_requests'].browse
+    return [(p, browse(ids)) for p, ids in env.cr.fetchall()]
+
+
+def stage_into(
+        branch: Branch,
+        batched_prs: List[PullRequests],
+        cleanup: contextlib.ExitStack,
+) -> Optional[Stagings]:
+    original_heads, staging_state = staging_setup(branch, batched_prs, cleanup)
+
+    staged = stage_batches(branch, batched_prs, staging_state)
+
+    if not staged:
+        return None
+
+    env = branch.env
+    heads = []
+    commits = []
+    for repo, it in staging_state.items():
+        if it.head != original_heads[repo]:
+            # if we staged something for that repo, just create a record for
+            # that commit, or flag existing one as to-recheck in case there are
+            # already statuses we want to propagate to the staging or something
+            env.cr.execute(
+                "INSERT INTO runbot_merge_commit (sha, to_check, statuses) "
+                "VALUES (%s, true, '{}') "
+                "ON CONFLICT (sha) DO UPDATE SET to_check=true "
+                "RETURNING id",
+                [it.head]
+            )
+            [commit] = [head] = env.cr.fetchone()
+        else:
+            # if we didn't stage anything for that repo, create a dummy commit
+            # (with a uniquifier to ensure we don't hit a previous version of
+            # the same) to ensure the staging head is new and we're building
+            # everything
+            tree = it.gh.commit(it.head)['tree']
+            uniquifier = base64.b64encode(os.urandom(12)).decode('ascii')
+            dummy_head = it.gh('post', 'git/commits', json={
+                'tree': tree['sha'],
+                'parents': [it.head],
+                'message': f'''\
+force rebuild
+
+uniquifier: {uniquifier}
+For-Commit-Id: {it.head}
+''',
+            }).json()['sha']
+            # see above, ideally we don't need to mark the real head as
+            # `to_check` because it's an old commit but `DO UPDATE` is necessary
+            # for `RETURNING` to work, and it doesn't really hurt (maybe)
+            env.cr.execute(
+                "INSERT INTO runbot_merge_commit (sha, to_check, statuses) "
+                "VALUES (%s, false, '{}'), (%s, true, '{}') "
+                "ON CONFLICT (sha) DO UPDATE SET to_check=true "
+                "RETURNING id",
+                [it.head, dummy_head]
+            )
+            ([commit], [head]) = env.cr.fetchall()
+            it.head = dummy_head
+
+        heads.append(fields.Command.create({
+            'repository_id': repo.id,
+            'commit_id': head,
+        }))
+        commits.append(fields.Command.create({
+            'repository_id': repo.id,
+            'commit_id': commit,
+        }))
+
+    # create actual staging object
+    st: Stagings = env['runbot_merge.stagings'].create({
+        'target': branch.id,
+        'batch_ids': [(4, batch.id, 0) for batch in staged],
+        'heads': heads,
+        'commits': commits,
+    })
+    # create staging branch from tmp
+    token = branch.project_id.github_token
+    for repo in branch.project_id.repo_ids.having_branch(branch):
+        it = staging_state[repo]
+        _logger.info(
+            "%s: create staging for %s:%s at %s",
+            branch.project_id.name, repo.name, branch.name,
+            it.head
+        )
+        refname = 'staging.{}'.format(branch.name)
+        it.gh.set_ref(refname, it.head)
+
+        i = count()
+        @utils.backoff(delays=WAIT_FOR_VISIBILITY, exc=TimeoutError)
+        def wait_for_visibility():
+            if check_visibility(repo, refname, it.head, token):
+                _logger.info(
+                    "[repo] updated %s:%s to %s: ok (at %d/%d)",
+                    repo.name, refname, it.head,
+                    next(i), len(WAIT_FOR_VISIBILITY)
+                )
+                return
+            _logger.warning(
+                "[repo] updated %s:%s to %s: failed (at %d/%d)",
+                repo.name, refname, it.head,
+                next(i), len(WAIT_FOR_VISIBILITY)
+            )
+            raise TimeoutError("Staged head not updated after %d seconds" % sum(WAIT_FOR_VISIBILITY))
+
+    _logger.info("Created staging %s (%s) to %s", st, ', '.join(
+        '%s[%s]' % (batch, batch.prs)
+        for batch in staged
+    ), st.target.name)
+    return st
+
+
+def staging_setup(
+        target: Branch,
+        batched_prs: List[PullRequests],
+        cleanup: contextlib.ExitStack
+) -> Tuple[Dict[Repository, str], StagingState]:
+    """Sets up the staging:
+
+    - stores baseline info
+    - creates tmp branch via gh API (to remove)
+    - generates working copy for each repository with the target branch
+    """
+    all_prs: PullRequests = target.env['runbot_merge.pull_requests'].concat(*batched_prs)
+    cache_dir = user_cache_dir('mergebot')
+    staging_state = {}
+    original_heads = {}
+    for repo in target.project_id.repo_ids.having_branch(target):
+        gh = repo.github()
+        head = gh.head(target.name)
+        # create tmp staging branch
+        gh.set_ref('tmp.{}'.format(target.name), head)
+
+        source = git.get_local(repo, 'github')
+        source.fetch(
+            git.source_url(repo, 'github'),
+            # a full refspec is necessary to ensure we actually fetch the ref
+            # (not just the commit it points to) and update it.
+            # `git fetch $remote $branch` seems to work locally, but it might
+            # be hooked only to "proper" remote-tracking branches
+            # (in `refs/remotes`), it doesn't seem to work here
+            f'+refs/heads/{target.name}:refs/heads/{target.name}',
+            *(pr.head for pr in all_prs if pr.repository == repo)
+        )
+        Path(cache_dir, repo.name).parent.mkdir(parents=True, exist_ok=True)
+        d = cleanup.enter_context(tempfile.TemporaryDirectory(
+            prefix=f'{repo.name}-{target.name}-staging',
+            dir=cache_dir,
+        ))
+        working_copy = source.clone(d, branch=target.name)
+        original_heads[repo] = head
+        staging_state[repo] = StagingSlice(gh=gh, head=head, working_copy=working_copy)
+
+    return original_heads, staging_state
+
+
+def stage_batches(branch: Branch, batched_prs: List[PullRequests], staging_state: StagingState) -> Stagings:
+    batch_limit = branch.project_id.batch_limit
+    env = branch.env
+    staged = env['runbot_merge.batch']
+    for batch in batched_prs:
+        if len(staged) >= batch_limit:
+            break
+
+        try:
+            staged |= stage_batch(env, batch, staging_state)
+        except exceptions.MergeError as e:
+            pr = e.args[0]
+            _logger.info("Failed to stage %s into %s", pr.display_name, branch.name, exc_info=True)
+            if not staged or isinstance(e, exceptions.Unmergeable):
+                if len(e.args) > 1 and e.args[1]:
+                    reason = e.args[1]
+                else:
+                    reason = e.__cause__ or e.__context__
+                # if the reason is a json document, assume it's a github error
+                # and try to extract the error message to give it to the user
+                with contextlib.suppress(Exception):
+                    reason = json.loads(str(reason))['message'].lower()
+
+                pr.state = 'error'
+                env.ref('runbot_merge.pr.merge.failed')._send(
+                    repository=pr.repository,
+                    pull_request=pr.number,
+                    format_args={'pr': pr, 'reason': reason, 'exc': e},
+                )
+    return staged
+
+def check_visibility(repo: Repository, branch_name: str, expected_head: str, token: str):
+    """ Checks the repository actual to see if the new / expected head is
+    now visible
+    """
+    # v1 protocol provides URL for ref discovery: https://github.com/git/git/blob/6e0cc6776106079ed4efa0cc9abace4107657abf/Documentation/technical/http-protocol.txt#L187
+    # for more complete client this is also the capabilities discovery and
+    # the "entry point" for the service
+    url = 'https://github.com/{}.git/info/refs?service=git-upload-pack'.format(repo.name)
+    with requests.get(url, stream=True, auth=(token, '')) as resp:
+        if not resp.ok:
+            return False
+        for head, ref in parse_refs_smart(resp.raw.read):
+            if ref != ('refs/heads/' + branch_name):
+                continue
+            return head == expected_head
+        return False
+
+
+refline = re.compile(rb'([\da-f]{40}) ([^\0\n]+)(\0.*)?\n?')
+ZERO_REF = b'0'*40
+
+def parse_refs_smart(read: Callable[[int], bytes]) -> Iterator[Tuple[str, str]]:
+    """ yields pkt-line data (bytes), or None for flush lines """
+    def read_line() -> Optional[bytes]:
+        length = int(read(4), 16)
+        if length == 0:
+            return None
+        return read(length - 4)
+
+    header = read_line()
+    assert header and header.rstrip() == b'# service=git-upload-pack', header
+    assert read_line() is None, "failed to find first flush line"
+    # read lines until second delimiter
+    for line in iter(read_line, None):
+        if line.startswith(ZERO_REF):
+            break # empty list (no refs)
+        m = refline.fullmatch(line)
+        assert m
+        yield m[1].decode(), m[2].decode()
+
+
+UNCHECKABLE = ['merge_method', 'overrides', 'draft']
+
+
+def stage_batch(env: api.Environment, prs: PullRequests, staging: StagingState) -> Batch:
+    """
+    Updates meta[*][head] on success
+    """
+    new_heads: Dict[PullRequests, str] = {}
+    pr_fields = env['runbot_merge.pull_requests']._fields
+    for pr in prs:
+        gh = staging[pr.repository].gh
+
+        _logger.info(
+            "Staging pr %s for target %s; method=%s",
+            pr.display_name, pr.target.name,
+            pr.merge_method or (pr.squash and 'single') or None
+        )
+
+        target = 'tmp.{}'.format(pr.target.name)
+        original_head = gh.head(target)
+        try:
+            try:
+                method, new_heads[pr] = stage(pr, gh, target, related_prs=(prs - pr))
+                _logger.info(
+                    "Staged pr %s to %s by %s: %s -> %s",
+                    pr.display_name, pr.target.name, method,
+                    original_head, new_heads[pr]
+                )
+            except Exception:
+                # reset the head which failed, as rebase() may have partially
+                # updated it (despite later steps failing)
+                gh.set_ref(target, original_head)
+                # then reset every previous update
+                for to_revert in new_heads.keys():
+                    it = staging[to_revert.repository]
+                    it.gh.set_ref('tmp.{}'.format(to_revert.target.name), it.head)
+                raise
+        except github.MergeError as e:
+            raise exceptions.MergeError(pr) from e
+        except exceptions.Mismatch as e:
+            diff = ''.join(Differ().compare(
+                list(format_for_difflib((n, v) for n, v, _ in e.args[1])),
+                list(format_for_difflib((n, v) for n, _, v in e.args[1])),
+            ))
+            _logger.info("data mismatch on %s:\n%s", pr.display_name, diff)
+            env.ref('runbot_merge.pr.staging.mismatch')._send(
+                repository=pr.repository,
+                pull_request=pr.number,
+                format_args={
+                    'pr': pr,
+                    'mismatch': ', '.join(pr_fields[f].string for f in e.args[0]),
+                    'diff': diff,
+                    'unchecked': ', '.join(pr_fields[f].string for f in UNCHECKABLE)
+                }
+            )
+            return env['runbot_merge.batch']
+
+    # update meta to new heads
+    for pr, head in new_heads.items():
+        staging[pr.repository].head = head
+    return env['runbot_merge.batch'].create({
+        'target': prs[0].target.id,
+        'prs': [(4, pr.id, 0) for pr in prs],
+    })
+
+def format_for_difflib(items: Iterator[Tuple[str, object]]) -> Iterator[str]:
+    """ Bit of a pain in the ass because difflib really wants
+    all lines to be newline-terminated, but not all values are
+    actual lines, and also needs to split multiline values.
+    """
+    for name, value in items:
+        yield name + ':\n'
+        value = str(value)
+        if not value.endswith('\n'):
+            value += '\n'
+        yield from value.splitlines(keepends=True)
+        yield '\n'
+
+
+Method = Literal['merge', 'rebase-merge', 'rebase-ff', 'squash']
+def stage(pr: PullRequests, gh: github.GH, target: str, related_prs: PullRequests) -> Tuple[Method, str]:
+    # nb: pr_commits is oldest to newest so pr.head is pr_commits[-1]
+    _, prdict = gh.pr(pr.number)
+    commits = prdict['commits']
+    method: Method = pr.merge_method or ('rebase-ff' if commits == 1 else None)
+    if commits > 50 and method.startswith('rebase'):
+        raise exceptions.Unmergeable(pr, "Rebasing 50 commits is too much.")
+    if commits > 250:
+        raise exceptions.Unmergeable(
+            pr, "Merging PRs of 250 or more commits is not supported "
+                "(https://developer.github.com/v3/pulls/#list-commits-on-a-pull-request)"
+        )
+    pr_commits = gh.commits(pr.number)
+    for c in pr_commits:
+        if not (c['commit']['author']['email'] and c['commit']['committer']['email']):
+            raise exceptions.Unmergeable(
+                pr,
+                f"All commits must have author and committer email, "
+                f"missing email on {c['sha']} indicates the authorship is "
+                f"most likely incorrect."
+            )
+
+    # sync and signal possibly missed updates
+    invalid = {}
+    diff = []
+    pr_head = pr_commits[-1]['sha']
+    if pr.head != pr_head:
+        invalid['head'] = pr_head
+        diff.append(('Head', pr.head, pr_head))
+
+    if pr.target.name != prdict['base']['ref']:
+        branch = pr.env['runbot_merge.branch'].with_context(active_test=False).search([
+            ('name', '=', prdict['base']['ref']),
+            ('project_id', '=', pr.repository.project_id.id),
+        ])
+        if not branch:
+            pr.unlink()
+            raise exceptions.Unmergeable(pr, "While staging, found this PR had been retargeted to an un-managed branch.")
+        invalid['target'] = branch.id
+        diff.append(('Target branch', pr.target.name, branch.name))
+
+    if pr.squash != commits == 1:
+        invalid['squash'] = commits == 1
+        diff.append(('Single commit', pr.squash, commits == 1))
+
+    msg = utils.make_message(prdict)
+    if pr.message != msg:
+        invalid['message'] = msg
+        diff.append(('Message', pr.message, msg))
+
+    if invalid:
+        pr.write({**invalid, 'state': 'opened', 'head': pr_head})
+        raise exceptions.Mismatch(invalid, diff)
+
+    if pr.reviewed_by and pr.reviewed_by.name == pr.reviewed_by.github_login:
+        # XXX: find other trigger(s) to sync github name?
+        gh_name = gh.user(pr.reviewed_by.github_login)['name']
+        if gh_name:
+            pr.reviewed_by.name = gh_name
+
+    match method:
+        case 'merge':
+            fn = stage_merge
+        case 'rebase-merge':
+            fn = stage_rebase_merge
+        case 'rebase-ff':
+            fn = stage_rebase_ff
+        case 'squash':
+            fn = stage_squash
+    return method, fn(pr, gh, target, pr_commits, related_prs=related_prs)
+
+def stage_squash(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
+    msg = pr._build_merge_message(pr, related_prs=related_prs)
+    authorship = {}
+
+    authors = {
+        (c['commit']['author']['name'], c['commit']['author']['email'])
+        for c in commits
+    }
+    if len(authors) == 1:
+        name, email = authors.pop()
+        authorship['author']  = {'name': name, 'email': email}
+    else:
+        msg.headers.extend(sorted(
+            ('Co-Authored-By', "%s <%s>" % author)
+            for author in authors
+        ))
+
+    committers = {
+        (c['commit']['committer']['name'], c['commit']['committer']['email'])
+        for c in commits
+    }
+    if len(committers) == 1:
+        name, email = committers.pop()
+        authorship['committer'] = {'name': name, 'email': email}
+    # should committers also be added to co-authors?
+
+    original_head = gh.head(target)
+    merge_tree = gh.merge(pr.head, target, 'temp merge')['tree']['sha']
+    head = gh('post', 'git/commits', json={
+        **authorship,
+        'message': str(msg),
+        'tree': merge_tree,
+        'parents': [original_head],
+    }).json()['sha']
+    gh.set_ref(target, head)
+
+    commits_map = {c['sha']: head for c in commits}
+    commits_map[''] = head
+    pr.commits_map = json.dumps(commits_map)
+
+    return head
+
+def stage_rebase_ff(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
+    # updates head commit with PR number (if necessary) then rebases
+    # on top of target
+    msg = pr._build_merge_message(commits[-1]['commit']['message'], related_prs=related_prs)
+    commits[-1]['commit']['message'] = str(msg)
+    add_self_references(pr, commits[:-1])
+    head, mapping = gh.rebase(pr.number, target, commits=commits)
+    pr.commits_map = json.dumps({**mapping, '': head})
+    return head
+
+def stage_rebase_merge(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str :
+    add_self_references(pr, commits)
+    h, mapping = gh.rebase(pr.number, target, reset=True, commits=commits)
+    msg = pr._build_merge_message(pr, related_prs=related_prs)
+    merge_head = gh.merge(h, target, str(msg))['sha']
+    pr.commits_map = json.dumps({**mapping, '': merge_head})
+    return merge_head
+
+def stage_merge(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
+    pr_head = commits[-1] # oldest to newest
+    base_commit = None
+    head_parents = {p['sha'] for p in pr_head['parents']}
+    if len(head_parents) > 1:
+        # look for parent(s?) of pr_head not in PR, means it's
+        # from target (so we merged target in pr)
+        merge = head_parents - {c['sha'] for c in commits}
+        external_parents = len(merge)
+        if external_parents > 1:
+            raise exceptions.Unmergeable(
+                "The PR head can only have one parent from the base branch "
+                "(not part of the PR itself), found %d: %s" % (
+                    external_parents,
+                    ', '.join(merge)
+                ))
+        if external_parents == 1:
+            [base_commit] = merge
+
+    commits_map = {c['sha']: c['sha'] for c in commits}
+    if base_commit:
+        # replicate pr_head with base_commit replaced by
+        # the current head
+        original_head = gh.head(target)
+        merge_tree = gh.merge(pr_head['sha'], target, 'temp merge')['tree']['sha']
+        new_parents = [original_head] + list(head_parents - {base_commit})
+        msg = pr._build_merge_message(pr_head['commit']['message'], related_prs=related_prs)
+        copy = gh('post', 'git/commits', json={
+            'message': str(msg),
+            'tree': merge_tree,
+            'author': pr_head['commit']['author'],
+            'committer': pr_head['commit']['committer'],
+            'parents': new_parents,
+        }).json()
+        gh.set_ref(target, copy['sha'])
+        # merge commit *and old PR head* map to the pr head replica
+        commits_map[''] = commits_map[pr_head['sha']] = copy['sha']
+        pr.commits_map = json.dumps(commits_map)
+        return copy['sha']
+    else:
+        # otherwise do a regular merge
+        msg = pr._build_merge_message(pr)
+        merge_head = gh.merge(pr.head, target, str(msg))['sha']
+        # and the merge commit is the normal merge head
+        commits_map[''] = merge_head
+        pr.commits_map = json.dumps(commits_map)
+        return merge_head
+
+def is_mentioned(message: Union[PullRequests, str], pr: PullRequests, *, full_reference: bool = False) -> bool:
+    """Returns whether ``pr`` is mentioned in ``message```
+    """
+    if full_reference:
+        pattern = fr'\b{re.escape(pr.display_name)}\b'
+    else:
+        repository = pr.repository.name  # .replace('/', '\\/')
+        pattern = fr'( |\b{repository})#{pr.number}\b'
+    return bool(re.search(pattern, message if isinstance(message, str) else message.message))
+
+def add_self_references(pr: PullRequests, commits: List[github.PrCommit]):
+    """Adds a footer reference to ``self`` to all ``commits`` if they don't
+    already refer to the PR.
+    """
+    for c in (c['commit'] for c in commits):
+        if not is_mentioned(c['message'], pr):
+            message = c['message']
+            m = Message.from_message(message)
+            m.headers.pop('Part-Of', None)
+            m.headers.add('Part-Of', pr.display_name)
+            c['message'] = str(m)
+
+BREAK = re.compile(r'''
+    [ ]{0,3} # 0-3 spaces of indentation
+    # followed by a sequence of three or more matching -, _, or * characters,
+    # each followed optionally by any number of spaces or tabs
+    # so needs to start with a _, - or *, then have at least 2 more such
+    # interspersed with any number of spaces or tabs
+    ([*_-])
+    ([ \t]*\1){2,}
+    [ \t]*
+''', flags=re.VERBOSE)
+SETEX_UNDERLINE = re.compile(r'''
+    [ ]{0,3} # no more than 3 spaces indentation
+    [-=]+ # a sequence of = characters or a sequence of - characters
+    [ ]* # any number of trailing spaces
+    # we don't care about "a line containing a single -" because we want to
+    # disambiguate SETEX headings from thematic breaks, and thematic breaks have
+    # 3+ -. Doesn't look like GH interprets `- - -` as a line so yay...
+''', flags=re.VERBOSE)
+HEADER = re.compile('([A-Za-z-]+): (.*)')
+class Message:
+    @classmethod
+    def from_message(cls, msg: Union[PullRequests, str]) -> 'Message':
+        in_headers = True
+        maybe_setex = None
+        # creating from PR message -> remove content following break
+        if isinstance(msg, str):
+            message, handle_break = (msg, False)
+        else:
+            message, handle_break = (msg.message, True)
+        headers = []
+        body: List[str] = []
+        # don't process the title (first line) of the commit message
+        lines = message.splitlines()
+        for line in reversed(lines[1:]):
+            if maybe_setex:
+                # NOTE: actually slightly more complicated: it's a SETEX heading
+                #       only if preceding line(s) can be interpreted as a
+                #       paragraph so e.g. a title followed by a line of dashes
+                #       would indeed be a break, but this should be good enough
+                #       for now, if we need more we'll need a full-blown
+                #       markdown parser probably
+                if line: # actually a SETEX title -> add underline to body then process current
+                    body.append(maybe_setex)
+                else: # actually break, remove body then process current
+                    body = []
+                maybe_setex = None
+
+            if not line:
+                if not in_headers and body and body[-1]:
+                    body.append(line)
+                continue
+
+            if handle_break and BREAK.fullmatch(line):
+                if SETEX_UNDERLINE.fullmatch(line):
+                    maybe_setex = line
+                else:
+                    body = []
+                continue
+
+            h = HEADER.fullmatch(line)
+            if h:
+                # c-a-b = special case from an existing test, not sure if actually useful?
+                if in_headers or h[1].lower() == 'co-authored-by':
+                    headers.append(h.groups())
+                    continue
+
+            body.append(line)
+            in_headers = False
+
+        # if there are non-title body lines, add a separation after the title
+        if body and body[-1]:
+            body.append('')
+        body.append(lines[0])
+        return cls('\n'.join(reversed(body)), Headers(reversed(headers)))
+
+    def __init__(self, body: str, headers: Optional[Headers] = None):
+        self.body = body
+        self.headers = headers or Headers()
+
+    def __setattr__(self, name, value):
+        # make sure stored body is always stripped
+        if name == 'body':
+            value = value and value.strip()
+        super().__setattr__(name, value)
+
+    def __str__(self):
+        if not self.headers:
+            return self.body + '\n'
+
+        with io.StringIO(self.body) as msg:
+            msg.write(self.body)
+            msg.write('\n\n')
+            # https://git.wiki.kernel.org/index.php/CommitMessageConventions
+            # seems to mostly use capitalised names (rather than title-cased)
+            keys = list(OrderedSet(k.capitalize() for k in self.headers.keys()))
+            # c-a-b must be at the very end otherwise github doesn't see it
+            keys.sort(key=lambda k: k == 'Co-authored-by')
+            for k in keys:
+                for v in self.headers.getlist(k):
+                    msg.write(k)
+                    msg.write(': ')
+                    msg.write(v)
+                    msg.write('\n')
+
+            return msg.getvalue()
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 58c642fa..57698426 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -182,7 +182,6 @@ def test_stage_match(env, project, repo_a, repo_b, config, page):
     assert 'Related: {}'.format(pr_b.display_name) in repo_a.commit('master').message
     assert 'Related: {}'.format(pr_a.display_name) in repo_b.commit('master').message
 
-    print(pr_a.batch_ids.read(['staging_id', 'prs']))
     # check that related PRs *still* link to one another after merge
     assert get_related_pr_labels(pr_page(page, prx_a)) == [pr_b.display_name]
     assert get_related_pr_labels(pr_page(page, prx_b)) == [pr_a.display_name]
@@ -1093,13 +1092,6 @@ def test_multi_project(env, make_repo, setreviewers, users, config,
     pr1_id = to_pr(env, pr1)
     pr2_id = to_pr(env, pr2)
 
-    print(
-        pr1.repo.name, pr1.number, pr1_id.display_name, pr1_id.label,
-        '\n',
-        pr2.repo.name, pr2.number, pr2_id.display_name, pr2_id.label,
-        flush=True,
-    )
-
     assert pr1_id.state == 'ready' and not pr1_id.blocked
     assert pr2_id.state == 'validated'
 

From 2fbbe3fcdb595f3411d6288a76d334c5df254005 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 22 Aug 2023 08:03:26 +0200
Subject: [PATCH 037/192] [ADD] runbot_merge: github identity for the mergebot

Necessary to create commits *as* the mergebot without going through
the github API. Copy of the improved version from forwardport. *Not*
an override, to avoid unnecessarily triggering one or the other which
is confusing and weird.
---
 runbot_merge/models/project.py              | 43 ++++++++++++++++++++-
 runbot_merge/views/runbot_merge_project.xml |  4 ++
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index 67222ea8..e828d29b 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -1,9 +1,11 @@
 import logging
 import re
 
+import requests
 import sentry_sdk
 
-from odoo import models, fields
+from odoo import models, fields, api
+from odoo.exceptions import UserError
 
 _logger = logging.getLogger(__name__)
 class Project(models.Model):
@@ -28,13 +30,14 @@ class Project(models.Model):
     )
 
     github_token = fields.Char("Github Token", required=True)
+    github_name = fields.Char(store=True, compute="_compute_identity")
+    github_email = fields.Char(store=True, compute="_compute_identity")
     github_prefix = fields.Char(
         required=True,
         default="hanson", # mergebot du bot du bot du~
         help="Prefix (~bot name) used when sending commands from PR "
              "comments e.g. [hanson retry] or [hanson r+ p=1]",
     )
-    github_name = fields.Char(related='github_prefix')
 
     batch_limit = fields.Integer(
         default=8, group_operator=None, help="Maximum number of PRs staged together")
@@ -48,6 +51,42 @@ class Project(models.Model):
     freeze_id = fields.Many2one('runbot_merge.project.freeze', compute='_compute_freeze')
     freeze_reminder = fields.Text()
 
+    @api.depends('github_token')
+    def _compute_identity(self):
+        s = requests.Session()
+        for project in self:
+            if not project.github_token or (project.github_name and project.github_email):
+                continue
+
+            r0 = s.get('https://api.github.com/user', headers={
+                'Authorization': 'token %s' % project.github_token
+            })
+            if not r0.ok:
+                _logger.error("Failed to fetch merge bot information for project %s: %s", project.name, r0.text or r0.content)
+                continue
+
+            r0 = r0.json()
+            project.github_name = r0['name'] or r0['login']
+            if email := r0['email']:
+                project.github_email = email
+                continue
+
+            if 'user:email' not in set(re.split(r',\s*', r0.headers['x-oauth-scopes'])):
+                raise UserError("The merge bot github token needs the user:email scope to fetch the bot's identity.")
+            r1 = s.get('https://api.github.com/user/emails', headers={
+                'Authorization': 'token %s' % project.github_token
+            })
+            if not r1.ok:
+                _logger.error("Failed to fetch merge bot emails for project %s: %s", project.name, r1.text or r1.content)
+                continue
+            project.github_email = next((
+                entry['email']
+                for entry in r1.json()
+                if entry['primary']
+            ), None)
+            if not project.github_email:
+                raise UserError("The merge bot needs a public or accessible primary email set up.")
+
     def _check_stagings(self, commit=False):
         # check branches with an active staging
         for branch in self.env['runbot_merge.branch']\
diff --git a/runbot_merge/views/runbot_merge_project.xml b/runbot_merge/views/runbot_merge_project.xml
index 544f9cf9..c131dc73 100644
--- a/runbot_merge/views/runbot_merge_project.xml
+++ b/runbot_merge/views/runbot_merge_project.xml
@@ -25,6 +25,10 @@
                     <group>
                         <group>
                             <field name="github_token"/>
+                            <field name="github_name" readonly="0"
+                                   help="Identity when creating new commits, defaults to github name, falls back to login."/>
+                            <field name="github_email" readonly="0"
+                                   help="Identity when creating new commits, defaults to public email, falls back to primary email."/>
                             <field name="secret"/>
                         </group>
                         <group>

From 85a78900239d793e3fd62dcfbf7914f9e223a5ea Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 18 Aug 2023 13:51:18 +0200
Subject: [PATCH 038/192] [CHG] runbot_merge: switch staging from github API to
 local

It has been a consideration for a while, but the pain of subtly
interacting with git via the ignominous CLI kept it back. Then ~~the
fire nation attacked~~ github got more and more tight-fisted (and in
some ways less reliable) with their API.

Staging pretty much just interacts with the git database, so it's both
a facultative github operator (it can just interact with git directly)
and a big consumer of API requests (because the git database endpoints
are very low level so it takes quite a bit of work to do anything
especially when high-level operations like rebase have to be
replicated by hand).

Furthermore, an issue has also been noticed which can be attributed to
using the github API (and that API's reliability getting worse): in
some cases github will fail to propagate a ref update / reset, so when
staging 2 PRs it's possible that the second one is merged on top of
the temporary branch of the first one, yielding a kinda broken commit
(in that it's a merge commit with a broken error message) instead of
the rebase / squash commit we expected.

As it turns out it's a very old issue but only happened very early so
was misattributed and not (sufficiently) guarded against:

- 41bd82244bb976bbd4d4be5e7bd792417c7dae6b (October 8th 2018) was
  spotted but thought to be a mergebot issue (might have been one of
  the opportunities where ref-checks were added though I can't find
  any reference to the commit in the runbot repo).
- 2be25052e147b151d1d8a5bc73cceb351586ce03 (October 15th, 2019) was
  missed (or ignored).
- 5a9fe7a7d05a9df7186072a7bffd60c6b428fd0e (July 31st, 2023) was
  spotted, but happened at a moment where everything kinda broke
  because of github rate-limiting ref updates, so the forensics were
  difficult and it was attributed to rate limiting issues.
- f10d03bf0f2e8f88f62a5d8356b84f714196130f (August 24th, 2023) broke
  the camel's back (and the head block): the logs were not too
  interspersed with other garbage and pretty clear that github ack'd a
  ref update, returned the correct oid when checking the ref, then
  returned the wrong oid when fetching it later on.

No Working Copy
===============

The working copy turns out to not be necessary, the plumbing commands
we *need* work just fine on a bare repository.

Working without a WC means we had to reimplement the high level
operations (rebase) by hand much as we'd done previously, *but* we
needed to do that anyway as git doesn't seem to provide any way to
retrieve the mapping when rebasing/cherrypicking, and cherrypicking by
commit doesn't work well as it can't really find the *merge base* it
needs.

Forward-porting can almost certainly be implemented similarly (with
some overhead), issue #803 has been opened to keep track of the idea.

No TMP
======

The `tmp.` branches are no more, the process of creating stagings is
based entirely around oids, if staging something fails we can just
abandon the oids (they'll be collected by the weekly GC), we only
need to update the staging branches at the very end of the process.

This simplifies things a fair bit.

For now we have stopped checking for visibility / backoff as we're
pushing via git, hopefully it is a more reliable reference than the
API.

Commmit Message Formatting
==========================

There's some unfortunate churn in the test, as the handling of
trailing newlines differs between github's APIs and git itself.

Fixes #247

PS: It might be a good idea to use pygit2 instead of the CLI
    eventually, the library is typed which is nice, and it avoids
    shelling out although that's really unlikely to be a major cost.
---
 conftest.py                            |   3 +
 mergebot_test_utils/utils.py           |   7 +-
 runbot_merge/git.py                    | 118 ++++++++++-
 runbot_merge/models/stagings_create.py | 271 +++++++++++--------------
 runbot_merge/tests/test_basic.py       | 104 +++++-----
 runbot_merge/tests/test_multirepo.py   |  12 +-
 6 files changed, 306 insertions(+), 209 deletions(-)

diff --git a/conftest.py b/conftest.py
index a244f2af..3d936e56 100644
--- a/conftest.py
+++ b/conftest.py
@@ -537,6 +537,9 @@ class Repo:
         self.hook = False
         repos.append(self)
 
+    def __repr__(self):
+        return f'<conftest.Repo {self.name}>'
+
     @property
     def owner(self):
         return self.name.split('/')[0]
diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index b87a2345..fae0d208 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -8,7 +8,8 @@ MESSAGE_TEMPLATE = """{message}
 
 closes {repo}#{number}
 
-{headers}Signed-off-by: {name} <{email}>"""
+{headers}Signed-off-by: {name} <{email}>
+"""
 # target branch '-' source branch '-' base64 unique '-fw'
 REF_PATTERN = r'{target}-{source}-[a-zA-Z0-9_-]{{4}}-fw'
 
@@ -136,6 +137,4 @@ def to_pr(env, pr):
     return pr
 
 def part_of(label, pr_id, *, separator='\n\n'):
-    """ Adds the "part-of" pseudo-header in the footer.
-    """
-    return f'{label}{separator}Part-of: {pr_id.display_name}'
+    return f'{label}{separator}Part-of: {pr_id.display_name}\n'
diff --git a/runbot_merge/git.py b/runbot_merge/git.py
index 91297f9e..09061933 100644
--- a/runbot_merge/git.py
+++ b/runbot_merge/git.py
@@ -1,12 +1,14 @@
 import dataclasses
 import itertools
 import logging
+import os
 import pathlib
 import resource
 import subprocess
-from typing import Optional, TypeVar
+from typing import Optional, TypeVar, Union, Sequence, Tuple, Dict
 
 from odoo.tools.appdirs import user_cache_dir
+from .github import MergeError, PrCommit
 
 _logger = logging.getLogger(__name__)
 
@@ -17,6 +19,7 @@ def source_url(repository, prefix: str) -> str:
         repository.name,
     )
 
+Authorship = Union[Tuple[str, str], Tuple[str, str, str]]
 
 def get_local(repository, prefix: Optional[str]) -> 'Optional[Repo]':
     repos_dir = pathlib.Path(user_cache_dir('mergebot'))
@@ -104,6 +107,119 @@ class Repo:
         )
         return Repo(to)
 
+    def rebase(self, dest: str, commits: Sequence[PrCommit]) -> Tuple[str, Dict[str, str]]:
+        """Implements rebase by hand atop plumbing so:
+
+        - we can work without a working copy
+        - we can track individual commits (and store the mapping)
+
+        It looks like `--merge-base` is not sufficient for `merge-tree` to
+        correctly keep track of history, so it loses contents. Therefore
+        implement in two passes as in the github version.
+        """
+        repo = self.stdout().with_config(text=True, check=False)
+
+        logger = _logger.getChild('rebase')
+        logger.debug("rebasing %s on %s (reset=%s, commits=%s)",
+                     self._repo, dest, len(commits))
+        if not commits:
+            raise MergeError("PR has no commits")
+
+        new_trees = []
+        parent = dest
+        for original in commits:
+            if len(original['parents']) != 1:
+                raise MergeError(
+                    f"commits with multiple parents ({original['sha']}) can not be rebased, "
+                    "either fix the branch to remove merges or merge without "
+                    "rebasing")
+
+            new_trees.append(check(repo.merge_tree(parent, original['sha'])).stdout.strip())
+            parent = check(repo.commit_tree(
+                tree=new_trees[-1],
+                parents=[parent, original['sha']],
+                message=f'temp rebase {original["sha"]}',
+            )).stdout.strip()
+
+        mapping = {}
+        for original, tree in zip(commits, new_trees):
+            authorship = check(repo.show('--no-patch', '--pretty="%an%n%ae%n%ai%n%cn%n%ce"', original['sha']))
+            author_name, author_email, author_date, committer_name, committer_email =\
+                authorship.stdout.splitlines()
+
+            c = check(repo.commit_tree(
+                tree=tree,
+                parents=[dest],
+                message=original['commit']['message'],
+                author=(author_name, author_email, author_date),
+                committer=(committer_name, committer_email),
+            )).stdout.strip()
+
+            logger.debug('copied %s to %s (parent: %s)', original['sha'], c, dest)
+            dest = mapping[original['sha']] = c
+
+        return dest, mapping
+
+    def merge(self, c1: str, c2: str, msg: str, *, author: Tuple[str, str]) -> str:
+        repo = self.stdout().with_config(text=True, check=False)
+
+        t = repo.merge_tree(c1, c2)
+        if t.returncode:
+            raise MergeError(t.stderr)
+
+        c = self.commit_tree(
+            tree=t.stdout.strip(),
+            message=msg,
+            parents=[c1, c2],
+            author=author,
+        )
+        if c.returncode:
+            raise MergeError(c.stderr)
+        return c.stdout.strip()
+
+    def commit_tree(
+        self, *, tree: str, message: str,
+        parents: Sequence[str] = (),
+        author: Optional[Authorship] = None,
+        committer: Optional[Authorship] = None,
+    ) -> subprocess.CompletedProcess:
+        authorship = {}
+        if author:
+            authorship['GIT_AUTHOR_NAME'] = author[0]
+            authorship['GIT_AUTHOR_EMAIL'] = author[1]
+            if len(author) > 2:
+                authorship['GIT_AUTHOR_DATE'] = author[2]
+        if committer:
+            authorship['GIT_COMMITTER_NAME'] = committer[0]
+            authorship['GIT_COMMITTER_EMAIL'] = committer[1]
+            if len(committer) > 2:
+                authorship['GIT_COMMITTER_DATE'] = committer[2]
+
+        return self.with_config(
+            stdout=subprocess.PIPE,
+            text=True,
+            env={
+                **os.environ,
+                **authorship,
+                # we don't want git to use the timezone of the machine it's
+                # running on: previously it used the timezone configured in
+                # github (?), which I think / assume defaults to a generic UTC
+                'TZ': 'UTC',
+            }
+        )._run(
+            'commit-tree',
+            tree,
+            '-m', message,
+            *itertools.chain.from_iterable(('-p', p) for p in parents),
+        )
+
+def check(p: subprocess.CompletedProcess) -> subprocess.CompletedProcess:
+    if not p.returncode:
+        return p
+
+    _logger.info("rebase failed at %s\nstdout:\n%s\nstderr:\n%s", p.args, p.stdout, p.stderr)
+    raise MergeError(p.stderr or 'merge conflict')
+
 
 @dataclasses.dataclass
 class GitCommand:
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index b00342dc..544712a7 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -6,10 +6,9 @@ import json
 import logging
 import os
 import re
-import tempfile
 from difflib import Differ
-from itertools import count, takewhile
-from pathlib import Path
+from itertools import takewhile
+from operator import itemgetter
 from typing import Dict, Union, Optional, Literal, Callable, Iterator, Tuple, List, TypeAlias
 
 import requests
@@ -17,7 +16,6 @@ from werkzeug.datastructures import Headers
 
 from odoo import api, models, fields
 from odoo.tools import OrderedSet
-from odoo.tools.appdirs import user_cache_dir
 from .pull_requests import Branch, Stagings, PullRequests, Repository, Batch
 from .. import exceptions, utils, github, git
 
@@ -40,7 +38,7 @@ class StagingSlice:
     """
     gh: github.GH
     head: str
-    working_copy: git.Repo
+    repo: git.Repo
 
 
 StagingState: TypeAlias = Dict[Repository, StagingSlice]
@@ -82,43 +80,7 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
     else: # p=2
         batched_prs = [pr_ids for _, pr_ids in takewhile(lambda r: r[0] == priority, rows)]
 
-    with contextlib.ExitStack() as cleanup:
-        return stage_into(branch, batched_prs, cleanup)
-
-
-def ready_prs(for_branch: Branch) -> List[Tuple[int, PullRequests]]:
-    env = for_branch.env
-    env.cr.execute("""
-    SELECT
-      min(pr.priority) as priority,
-      array_agg(pr.id) AS match
-    FROM runbot_merge_pull_requests pr
-    WHERE pr.target = any(%s)
-      -- exclude terminal states (so there's no issue when
-      -- deleting branches & reusing labels)
-      AND pr.state != 'merged'
-      AND pr.state != 'closed'
-    GROUP BY
-        pr.target,
-        CASE
-            WHEN pr.label SIMILAR TO '%%:patch-[[:digit:]]+'
-                THEN pr.id::text
-            ELSE pr.label
-        END
-    HAVING
-        bool_or(pr.state = 'ready') or bool_or(pr.priority = 0)
-    ORDER BY min(pr.priority), min(pr.id)
-    """, [for_branch.ids])
-    browse = env['runbot_merge.pull_requests'].browse
-    return [(p, browse(ids)) for p, ids in env.cr.fetchall()]
-
-
-def stage_into(
-        branch: Branch,
-        batched_prs: List[PullRequests],
-        cleanup: contextlib.ExitStack,
-) -> Optional[Stagings]:
-    original_heads, staging_state = staging_setup(branch, batched_prs, cleanup)
+    original_heads, staging_state = staging_setup(branch, batched_prs)
 
     staged = stage_batches(branch, batched_prs, staging_state)
 
@@ -146,18 +108,22 @@ def stage_into(
             # (with a uniquifier to ensure we don't hit a previous version of
             # the same) to ensure the staging head is new and we're building
             # everything
-            tree = it.gh.commit(it.head)['tree']
+            project = branch.project_id
             uniquifier = base64.b64encode(os.urandom(12)).decode('ascii')
-            dummy_head = it.gh('post', 'git/commits', json={
-                'tree': tree['sha'],
-                'parents': [it.head],
-                'message': f'''\
+            dummy_head = it.repo.with_config(check=True).commit_tree(
+                # somewhat exceptionally, `commit-tree` wants an actual tree
+                # not a tree-ish
+                tree=f'{it.head}^{{tree}}',
+                parents=[it.head],
+                author=(project.github_name, project.github_email),
+                message=f'''\
 force rebuild
 
 uniquifier: {uniquifier}
 For-Commit-Id: {it.head}
 ''',
-            }).json()['sha']
+            ).stdout.strip()
+
             # see above, ideally we don't need to mark the real head as
             # `to_check` because it's an old commit but `DO UPDATE` is necessary
             # for `RETURNING` to work, and it doesn't really hurt (maybe)
@@ -187,34 +153,17 @@ For-Commit-Id: {it.head}
         'heads': heads,
         'commits': commits,
     })
-    # create staging branch from tmp
-    token = branch.project_id.github_token
-    for repo in branch.project_id.repo_ids.having_branch(branch):
-        it = staging_state[repo]
+    for repo, it in staging_state.items():
         _logger.info(
             "%s: create staging for %s:%s at %s",
             branch.project_id.name, repo.name, branch.name,
             it.head
         )
-        refname = 'staging.{}'.format(branch.name)
-        it.gh.set_ref(refname, it.head)
-
-        i = count()
-        @utils.backoff(delays=WAIT_FOR_VISIBILITY, exc=TimeoutError)
-        def wait_for_visibility():
-            if check_visibility(repo, refname, it.head, token):
-                _logger.info(
-                    "[repo] updated %s:%s to %s: ok (at %d/%d)",
-                    repo.name, refname, it.head,
-                    next(i), len(WAIT_FOR_VISIBILITY)
-                )
-                return
-            _logger.warning(
-                "[repo] updated %s:%s to %s: failed (at %d/%d)",
-                repo.name, refname, it.head,
-                next(i), len(WAIT_FOR_VISIBILITY)
-            )
-            raise TimeoutError("Staged head not updated after %d seconds" % sum(WAIT_FOR_VISIBILITY))
+        it.repo.stdout(False).check(True).push(
+            '-f',
+            git.source_url(repo, 'github'),
+            f'{it.head}:refs/heads/staging.{branch.name}',
+        )
 
     _logger.info("Created staging %s (%s) to %s", st, ', '.join(
         '%s[%s]' % (batch, batch.prs)
@@ -223,10 +172,36 @@ For-Commit-Id: {it.head}
     return st
 
 
+def ready_prs(for_branch: Branch) -> List[Tuple[int, PullRequests]]:
+    env = for_branch.env
+    env.cr.execute("""
+    SELECT
+      min(pr.priority) as priority,
+      array_agg(pr.id) AS match
+    FROM runbot_merge_pull_requests pr
+    WHERE pr.target = any(%s)
+      -- exclude terminal states (so there's no issue when
+      -- deleting branches & reusing labels)
+      AND pr.state != 'merged'
+      AND pr.state != 'closed'
+    GROUP BY
+        pr.target,
+        CASE
+            WHEN pr.label SIMILAR TO '%%:patch-[[:digit:]]+'
+                THEN pr.id::text
+            ELSE pr.label
+        END
+    HAVING
+        bool_or(pr.state = 'ready') or bool_or(pr.priority = 0)
+    ORDER BY min(pr.priority), min(pr.id)
+    """, [for_branch.ids])
+    browse = env['runbot_merge.pull_requests'].browse
+    return [(p, browse(ids)) for p, ids in env.cr.fetchall()]
+
+
 def staging_setup(
         target: Branch,
         batched_prs: List[PullRequests],
-        cleanup: contextlib.ExitStack
 ) -> Tuple[Dict[Repository, str], StagingState]:
     """Sets up the staging:
 
@@ -235,14 +210,11 @@ def staging_setup(
     - generates working copy for each repository with the target branch
     """
     all_prs: PullRequests = target.env['runbot_merge.pull_requests'].concat(*batched_prs)
-    cache_dir = user_cache_dir('mergebot')
     staging_state = {}
     original_heads = {}
     for repo in target.project_id.repo_ids.having_branch(target):
         gh = repo.github()
         head = gh.head(target.name)
-        # create tmp staging branch
-        gh.set_ref('tmp.{}'.format(target.name), head)
 
         source = git.get_local(repo, 'github')
         source.fetch(
@@ -255,14 +227,8 @@ def staging_setup(
             f'+refs/heads/{target.name}:refs/heads/{target.name}',
             *(pr.head for pr in all_prs if pr.repository == repo)
         )
-        Path(cache_dir, repo.name).parent.mkdir(parents=True, exist_ok=True)
-        d = cleanup.enter_context(tempfile.TemporaryDirectory(
-            prefix=f'{repo.name}-{target.name}-staging',
-            dir=cache_dir,
-        ))
-        working_copy = source.clone(d, branch=target.name)
         original_heads[repo] = head
-        staging_state[repo] = StagingSlice(gh=gh, head=head, working_copy=working_copy)
+        staging_state[repo] = StagingSlice(gh=gh, head=head, repo=source.stdout().with_config(text=True, check=False))
 
     return original_heads, staging_state
 
@@ -343,39 +309,29 @@ UNCHECKABLE = ['merge_method', 'overrides', 'draft']
 
 
 def stage_batch(env: api.Environment, prs: PullRequests, staging: StagingState) -> Batch:
-    """
-    Updates meta[*][head] on success
+    """Stages the batch represented by the ``prs`` recordset, onto the
+    current corresponding staging heads.
+
+    Alongside returning the newly created batch, updates ``staging[*].head``
+    in-place on success. On failure, the heads should not be touched.
     """
     new_heads: Dict[PullRequests, str] = {}
     pr_fields = env['runbot_merge.pull_requests']._fields
     for pr in prs:
-        gh = staging[pr.repository].gh
-
+        info = staging[pr.repository]
         _logger.info(
             "Staging pr %s for target %s; method=%s",
             pr.display_name, pr.target.name,
             pr.merge_method or (pr.squash and 'single') or None
         )
 
-        target = 'tmp.{}'.format(pr.target.name)
-        original_head = gh.head(target)
         try:
-            try:
-                method, new_heads[pr] = stage(pr, gh, target, related_prs=(prs - pr))
-                _logger.info(
-                    "Staged pr %s to %s by %s: %s -> %s",
-                    pr.display_name, pr.target.name, method,
-                    original_head, new_heads[pr]
-                )
-            except Exception:
-                # reset the head which failed, as rebase() may have partially
-                # updated it (despite later steps failing)
-                gh.set_ref(target, original_head)
-                # then reset every previous update
-                for to_revert in new_heads.keys():
-                    it = staging[to_revert.repository]
-                    it.gh.set_ref('tmp.{}'.format(to_revert.target.name), it.head)
-                raise
+            method, new_heads[pr] = stage(pr, info, related_prs=(prs - pr))
+            _logger.info(
+                "Staged pr %s to %s by %s: %s -> %s",
+                pr.display_name, pr.target.name, method,
+                info.head, new_heads[pr]
+            )
         except github.MergeError as e:
             raise exceptions.MergeError(pr) from e
         except exceptions.Mismatch as e:
@@ -419,9 +375,9 @@ def format_for_difflib(items: Iterator[Tuple[str, object]]) -> Iterator[str]:
 
 
 Method = Literal['merge', 'rebase-merge', 'rebase-ff', 'squash']
-def stage(pr: PullRequests, gh: github.GH, target: str, related_prs: PullRequests) -> Tuple[Method, str]:
+def stage(pr: PullRequests, info: StagingSlice, related_prs: PullRequests) -> Tuple[Method, str]:
     # nb: pr_commits is oldest to newest so pr.head is pr_commits[-1]
-    _, prdict = gh.pr(pr.number)
+    _, prdict = info.gh.pr(pr.number)
     commits = prdict['commits']
     method: Method = pr.merge_method or ('rebase-ff' if commits == 1 else None)
     if commits > 50 and method.startswith('rebase'):
@@ -431,7 +387,7 @@ def stage(pr: PullRequests, gh: github.GH, target: str, related_prs: PullRequest
             pr, "Merging PRs of 250 or more commits is not supported "
                 "(https://developer.github.com/v3/pulls/#list-commits-on-a-pull-request)"
         )
-    pr_commits = gh.commits(pr.number)
+    pr_commits = info.gh.commits(pr.number)
     for c in pr_commits:
         if not (c['commit']['author']['email'] and c['commit']['committer']['email']):
             raise exceptions.Unmergeable(
@@ -475,7 +431,7 @@ def stage(pr: PullRequests, gh: github.GH, target: str, related_prs: PullRequest
 
     if pr.reviewed_by and pr.reviewed_by.name == pr.reviewed_by.github_login:
         # XXX: find other trigger(s) to sync github name?
-        gh_name = gh.user(pr.reviewed_by.github_login)['name']
+        gh_name = info.gh.user(pr.reviewed_by.github_login)['name']
         if gh_name:
             pr.reviewed_by.name = gh_name
 
@@ -488,43 +444,46 @@ def stage(pr: PullRequests, gh: github.GH, target: str, related_prs: PullRequest
             fn = stage_rebase_ff
         case 'squash':
             fn = stage_squash
-    return method, fn(pr, gh, target, pr_commits, related_prs=related_prs)
+    return method, fn(pr, info, pr_commits, related_prs=related_prs)
 
-def stage_squash(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
+def stage_squash(pr: PullRequests, info: StagingSlice, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
     msg = pr._build_merge_message(pr, related_prs=related_prs)
-    authorship = {}
 
     authors = {
         (c['commit']['author']['name'], c['commit']['author']['email'])
         for c in commits
     }
     if len(authors) == 1:
-        name, email = authors.pop()
-        authorship['author']  = {'name': name, 'email': email}
+        author = authors.pop()
     else:
         msg.headers.extend(sorted(
             ('Co-Authored-By', "%s <%s>" % author)
             for author in authors
         ))
+        author = (pr.repository.project_id.github_name, pr.repository.project_id.github_email)
 
     committers = {
         (c['commit']['committer']['name'], c['commit']['committer']['email'])
         for c in commits
     }
-    if len(committers) == 1:
-        name, email = committers.pop()
-        authorship['committer'] = {'name': name, 'email': email}
     # should committers also be added to co-authors?
+    committer = committers.pop() if len(committers) == 1 else None
 
-    original_head = gh.head(target)
-    merge_tree = gh.merge(pr.head, target, 'temp merge')['tree']['sha']
-    head = gh('post', 'git/commits', json={
-        **authorship,
-        'message': str(msg),
-        'tree': merge_tree,
-        'parents': [original_head],
-    }).json()['sha']
-    gh.set_ref(target, head)
+    r = info.repo.merge_tree(info.head, pr.head)
+    if r.returncode:
+        raise exceptions.MergeError(pr, r.stderr)
+    merge_tree = r.stdout.strip()
+
+    r = info.repo.commit_tree(
+        tree=merge_tree,
+        parents=[info.head],
+        message=str(msg),
+        author=author,
+        committer=committer or author,
+    )
+    if r.returncode:
+        raise exceptions.MergeError(pr, r.stderr)
+    head = r.stdout.strip()
 
     commits_map = {c['sha']: head for c in commits}
     commits_map[''] = head
@@ -532,25 +491,30 @@ def stage_squash(pr: PullRequests, gh: github.GH, target: str, commits: List[git
 
     return head
 
-def stage_rebase_ff(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
+def stage_rebase_ff(pr: PullRequests, info: StagingSlice, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
     # updates head commit with PR number (if necessary) then rebases
     # on top of target
     msg = pr._build_merge_message(commits[-1]['commit']['message'], related_prs=related_prs)
     commits[-1]['commit']['message'] = str(msg)
     add_self_references(pr, commits[:-1])
-    head, mapping = gh.rebase(pr.number, target, commits=commits)
+    head, mapping = info.repo.rebase(info.head, commits=commits)
     pr.commits_map = json.dumps({**mapping, '': head})
     return head
 
-def stage_rebase_merge(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str :
+def stage_rebase_merge(pr: PullRequests, info: StagingSlice, commits: List[github.PrCommit], related_prs: PullRequests) -> str :
     add_self_references(pr, commits)
-    h, mapping = gh.rebase(pr.number, target, reset=True, commits=commits)
+    h, mapping = info.repo.rebase(info.head, commits=commits)
     msg = pr._build_merge_message(pr, related_prs=related_prs)
-    merge_head = gh.merge(h, target, str(msg))['sha']
+
+    project = pr.repository.project_id
+    merge_head= info.repo.merge(
+        info.head, h, str(msg),
+        author=(project.github_name, project.github_email),
+    )
     pr.commits_map = json.dumps({**mapping, '': merge_head})
     return merge_head
 
-def stage_merge(pr: PullRequests, gh: github.GH, target: str, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
+def stage_merge(pr: PullRequests, info: StagingSlice, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
     pr_head = commits[-1] # oldest to newest
     base_commit = None
     head_parents = {p['sha'] for p in pr_head['parents']}
@@ -573,26 +537,37 @@ def stage_merge(pr: PullRequests, gh: github.GH, target: str, commits: List[gith
     if base_commit:
         # replicate pr_head with base_commit replaced by
         # the current head
-        original_head = gh.head(target)
-        merge_tree = gh.merge(pr_head['sha'], target, 'temp merge')['tree']['sha']
-        new_parents = [original_head] + list(head_parents - {base_commit})
+        t = info.repo.merge_tree(info.head, pr_head['sha'])
+        if t.returncode:
+            raise exceptions.MergeError(pr, t.stderr)
+        merge_tree = t.stdout.strip()
+        new_parents = [info.head] + list(head_parents - {base_commit})
         msg = pr._build_merge_message(pr_head['commit']['message'], related_prs=related_prs)
-        copy = gh('post', 'git/commits', json={
-            'message': str(msg),
-            'tree': merge_tree,
-            'author': pr_head['commit']['author'],
-            'committer': pr_head['commit']['committer'],
-            'parents': new_parents,
-        }).json()
-        gh.set_ref(target, copy['sha'])
+
+        d2t = itemgetter('name', 'email', 'date')
+        c = info.repo.commit_tree(
+            tree=merge_tree,
+            parents=new_parents,
+            message=str(msg),
+            author=d2t(pr_head['commit']['author']),
+            committer=d2t(pr_head['commit']['committer']),
+        )
+        if c.returncode:
+            raise exceptions.MergeError(pr, c.stderr)
+        copy = c.stdout.strip()
+
         # merge commit *and old PR head* map to the pr head replica
-        commits_map[''] = commits_map[pr_head['sha']] = copy['sha']
+        commits_map[''] = commits_map[pr_head['sha']] = copy
         pr.commits_map = json.dumps(commits_map)
-        return copy['sha']
+        return copy
     else:
         # otherwise do a regular merge
         msg = pr._build_merge_message(pr)
-        merge_head = gh.merge(pr.head, target, str(msg))['sha']
+        project = pr.repository.project_id
+        merge_head = info.repo.merge(
+            info.head, pr.head, str(msg),
+            author=(project.github_name, project.github_email),
+        )
         # and the merge commit is the normal merge head
         commits_map[''] = merge_head
         pr.commits_map = json.dumps(commits_map)
@@ -707,10 +682,10 @@ class Message:
 
     def __str__(self):
         if not self.headers:
-            return self.body + '\n'
+            return self.body.rstrip() + '\n'
 
-        with io.StringIO(self.body) as msg:
-            msg.write(self.body)
+        with io.StringIO() as msg:
+            msg.write(self.body.rstrip())
             msg.write('\n\n')
             # https://git.wiki.kernel.org/index.php/CommitMessageConventions
             # seems to mostly use capitalised names (rather than title-cased)
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index ccc8e36f..815e6c8b 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -1,7 +1,6 @@
 import datetime
 import itertools
 import json
-import textwrap
 import time
 from unittest import mock
 
@@ -124,7 +123,7 @@ def test_trivial_flow(env, repo, page, users, config):
         'b': 'a second file',
     }
     assert master.message == "gibberish\n\nblahblah\n\ncloses {repo.name}#1"\
-                             "\n\nSigned-off-by: {reviewer.formatted_email}"\
+                             "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
                              .format(repo=repo, reviewer=get_partner(env, users['reviewer']))
 
 class TestCommitMessage:
@@ -149,7 +148,7 @@ class TestCommitMessage:
 
         master = repo.commit('heads/master')
         assert master.message == "simple commit message\n\ncloses {repo.name}#1"\
-                                 "\n\nSigned-off-by: {reviewer.formatted_email}"\
+                                 "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
                                  .format(repo=repo, reviewer=get_partner(env, users['reviewer']))
 
     def test_commit_existing(self, env, repo, users, config):
@@ -174,7 +173,7 @@ class TestCommitMessage:
         master = repo.commit('heads/master')
         # closes #1 is already present, should not modify message
         assert master.message == "simple commit message that closes #1"\
-                                 "\n\nSigned-off-by: {reviewer.formatted_email}"\
+                                 "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
                                  .format(reviewer=get_partner(env, users['reviewer']))
 
     def test_commit_other(self, env, repo, users, config):
@@ -199,7 +198,7 @@ class TestCommitMessage:
         master = repo.commit('heads/master')
         # closes on another repositoy, should modify the commit message
         assert master.message == "simple commit message that closes odoo/enterprise#1\n\ncloses {repo.name}#1"\
-                                 "\n\nSigned-off-by: {reviewer.formatted_email}"\
+                                 "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
                                  .format(repo=repo, reviewer=get_partner(env, users['reviewer']))
 
     def test_commit_wrong_number(self, env, repo, users, config):
@@ -224,7 +223,7 @@ class TestCommitMessage:
         master = repo.commit('heads/master')
         # closes on another repositoy, should modify the commit message
         assert master.message == "simple commit message that closes #11\n\ncloses {repo.name}#1"\
-                                 "\n\nSigned-off-by: {reviewer.formatted_email}"\
+                                 "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
                                  .format(repo=repo, reviewer=get_partner(env, users['reviewer']))
 
     def test_commit_delegate(self, env, repo, users, config):
@@ -254,7 +253,7 @@ class TestCommitMessage:
 
         master = repo.commit('heads/master')
         assert master.message == "simple commit message\n\ncloses {repo.name}#1"\
-                                 "\n\nSigned-off-by: {reviewer.formatted_email}"\
+                                 "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
                                  .format(repo=repo, reviewer=get_partner(env, users['other']))
 
     def test_commit_coauthored(self, env, repo, users, config):
@@ -292,7 +291,8 @@ Fixes a thing
 closes {repo.name}#1
 
 Signed-off-by: {reviewer.formatted_email}
-Co-authored-by: Bob <bob@example.com>""".format(
+Co-authored-by: Bob <bob@example.com>
+""".format(
             repo=repo,
             reviewer=get_partner(env, users['reviewer'])
         )
@@ -701,9 +701,9 @@ def test_ff_failure_batch(env, repo, users, config):
     reviewer = get_partner(env, users["reviewer"]).formatted_email
     assert messages == {
         'initial', 'NO!',
-        part_of('a1', pr_a), part_of('a2', pr_a), f'A\n\ncloses {pr_a.display_name}\n\nSigned-off-by: {reviewer}',
-        part_of('b1', pr_b), part_of('b2', pr_b), f'B\n\ncloses {pr_b.display_name}\n\nSigned-off-by: {reviewer}',
-        part_of('c1', pr_c), part_of('c2', pr_c), f'C\n\ncloses {pr_c.display_name}\n\nSigned-off-by: {reviewer}',
+        part_of('a1', pr_a), part_of('a2', pr_a), f'A\n\ncloses {pr_a.display_name}\n\nSigned-off-by: {reviewer}\n',
+        part_of('b1', pr_b), part_of('b2', pr_b), f'B\n\ncloses {pr_b.display_name}\n\nSigned-off-by: {reviewer}\n',
+        part_of('c1', pr_c), part_of('c2', pr_c), f'C\n\ncloses {pr_c.display_name}\n\nSigned-off-by: {reviewer}\n',
     }
 
 class TestPREdition:
@@ -1532,7 +1532,7 @@ commits, I need to know how to merge it:
         nb1 = node(part_of('B1', pr_id), node(part_of('B0', pr_id), nm2))
         reviewer = get_partner(env, users["reviewer"]).formatted_email
         merge_head = (
-            f'title\n\nbody\n\ncloses {pr_id.display_name}\n\nSigned-off-by: {reviewer}',
+            f'title\n\nbody\n\ncloses {pr_id.display_name}\n\nSigned-off-by: {reviewer}\n',
             frozenset([nm2, nb1])
         )
         assert staging == merge_head
@@ -1628,7 +1628,7 @@ commits, I need to know how to merge it:
         # then compare to the dag version of the right graph
         nm2 = node('M2', node('M1', node('M0')))
         reviewer = get_partner(env, users["reviewer"]).formatted_email
-        nb1 = node(f'B1\n\ncloses {pr_id.display_name}\n\nSigned-off-by: {reviewer}',
+        nb1 = node(f'B1\n\ncloses {pr_id.display_name}\n\nSigned-off-by: {reviewer}\n',
                    node(part_of('B0', pr_id), nm2))
         assert staging == nb1
 
@@ -1718,7 +1718,7 @@ commits, I need to know how to merge it:
         c0 = node('C0', m)
         reviewer = get_partner(env, users["reviewer"]).formatted_email
         expected = node('gibberish\n\nblahblah\n\ncloses {}#{}'
-                        '\n\nSigned-off-by: {}'.format(repo.name, prx.number, reviewer), m, c0)
+                        '\n\nSigned-off-by: {}\n'.format(repo.name, prx.number, reviewer), m, c0)
         assert log_to_node(repo.log('heads/master')), expected
         pr = env['runbot_merge.pull_requests'].search([
             ('repository.name', '=', repo.name),
@@ -1760,7 +1760,7 @@ commits, I need to know how to merge it:
         c0 = node('C0', m)
         reviewer = get_partner(env, users["reviewer"]).formatted_email
         expected = node('gibberish\n\ncloses {}#{}'
-                        '\n\nSigned-off-by: {}'.format(repo.name, prx.number, reviewer), m, c0)
+                        '\n\nSigned-off-by: {}\n'.format(repo.name, prx.number, reviewer), m, c0)
         assert log_to_node(repo.log('heads/master')), expected
 
     @pytest.mark.parametrize('separator', [
@@ -1790,15 +1790,15 @@ commits, I need to know how to merge it:
         env.run_crons()
 
         head = repo.commit('heads/master')
-        assert head.message == textwrap.dedent(f"""\
-        title
+        assert head.message == f"""\
+title
 
-        first
+first
 
-        closes {repo.name}#{pr.number}
+closes {repo.name}#{pr.number}
 
-        Signed-off-by: {reviewer}
-        """).strip(), "should not contain the content which follows the thematic break"
+Signed-off-by: {reviewer}
+""", "should not contain the content which follows the thematic break"
 
     def test_pr_message_setex_title(self, repo, env, users, config):
         """ should not break on a proper SETEX-style title """
@@ -1830,21 +1830,21 @@ removed
         env.run_crons()
 
         head = repo.commit('heads/master')
-        assert head.message == textwrap.dedent(f"""\
-        title
+        assert head.message == f"""\
+title
 
-        Title
-        ---
-        This is some text
+Title
+---
+This is some text
 
-        Title 2
-        -------
-        This is more text
+Title 2
+-------
+This is more text
 
-        closes {repo.name}#{pr.number}
+closes {repo.name}#{pr.number}
 
-        Signed-off-by: {reviewer}
-        """).strip(), "should not break the SETEX titles"
+Signed-off-by: {reviewer}
+""", "should not break the SETEX titles"
 
     def test_rebase_no_edit(self, repo, env, users, config):
         """ Only the merge messages should be de-breaked
@@ -1867,17 +1867,17 @@ removed
         env.run_crons()
 
         head = repo.commit('heads/master')
-        assert head.message == textwrap.dedent(f"""\
-        Commit
+        assert head.message == f"""\
+Commit
 
-        first
-        ***
-        second
+first
+***
+second
 
-        closes {repo.name}#{pr.number}
+closes {repo.name}#{pr.number}
 
-        Signed-off-by: {reviewer}
-        """).strip(), "squashed / rebased messages should not be stripped"
+Signed-off-by: {reviewer}
+""", "squashed / rebased messages should not be stripped"
 
     def test_title_no_edit(self, repo, env, users, config):
         """The first line of a commit message should not be taken in account for
@@ -1909,13 +1909,15 @@ thing: thong
 
 closes {pr_id.display_name}
 
-Signed-off-by: {reviewer}"""
+Signed-off-by: {reviewer}
+"""
         assert repo.commit(staging_head.parents[0]).message == f"""\
 Some: thing
 
 is odd
 
-Part-of: {pr_id.display_name}"""
+Part-of: {pr_id.display_name}
+"""
 
     def test_pr_mergehead(self, repo, env, config):
         """ if the head of the PR is a merge commit and one of the parents is
@@ -2000,7 +2002,7 @@ Part-of: {pr_id.display_name}"""
         m1 = node('M1')
         reviewer = get_partner(env, users["reviewer"]).formatted_email
         expected = node(
-            'T\n\nTT\n\ncloses {}#{}\n\nSigned-off-by: {}'.format(repo.name, prx.number, reviewer),
+            'T\n\nTT\n\ncloses {}#{}\n\nSigned-off-by: {}\n'.format(repo.name, prx.number, reviewer),
             node('M2', m1),
             node('C1', node('C0', m1), node('B0', m1))
         )
@@ -2076,7 +2078,7 @@ Part-of: {pr_id.display_name}"""
 
 closes {pr1_id.display_name}
 
-Signed-off-by: {get_partner(env, users["reviewer"]).formatted_email}\
+Signed-off-by: {get_partner(env, users["reviewer"]).formatted_email}
 """
         assert one['commit']['committer']['name'] == a_user['name']
         assert one['commit']['committer']['email'] == a_user['email']
@@ -2107,7 +2109,7 @@ closes {pr2_id.display_name}
 
 Signed-off-by: {get_partner(env, users["reviewer"]).formatted_email}
 Co-authored-by: {a_user['name']} <{a_user['email']}>
-Co-authored-by: {other_user['name']} <{other_user['email']}>\
+Co-authored-by: {other_user['name']} <{other_user['email']}>
 """
         assert repo.read_tree(repo.commit(two['sha'])) == {
             'a': '0',
@@ -2667,12 +2669,12 @@ class TestBatching(object):
         staging = log_to_node(log)
         reviewer = get_partner(env, users["reviewer"]).formatted_email
         p1 = node(
-            'title PR1\n\nbody PR1\n\ncloses {}\n\nSigned-off-by: {}'.format(pr1.display_name, reviewer),
+            'title PR1\n\nbody PR1\n\ncloses {}\n\nSigned-off-by: {}\n'.format(pr1.display_name, reviewer),
             node('initial'),
             node(part_of('commit_PR1_01', pr1), node(part_of('commit_PR1_00', pr1), node('initial')))
         )
         p2 = node(
-            'title PR2\n\nbody PR2\n\ncloses {}\n\nSigned-off-by: {}'.format(pr2.display_name, reviewer),
+            'title PR2\n\nbody PR2\n\ncloses {}\n\nSigned-off-by: {}\n'.format(pr2.display_name, reviewer),
             p1,
             node(part_of('commit_PR2_01', pr2), node(part_of('commit_PR2_00', pr2), p1))
         )
@@ -2707,12 +2709,12 @@ class TestBatching(object):
         reviewer = get_partner(env, users["reviewer"]).formatted_email
 
         p1 = node(
-            'title PR1\n\nbody PR1\n\ncloses {}#{}\n\nSigned-off-by: {}'.format(repo.name, pr1.number, reviewer),
+            'title PR1\n\nbody PR1\n\ncloses {}#{}\n\nSigned-off-by: {}\n'.format(repo.name, pr1.number, reviewer),
             node('initial'),
             node('commit_PR1_01', node('commit_PR1_00', node('initial')))
         )
         p2 = node(
-            'title PR2\n\nbody PR2\n\ncloses {}#{}\n\nSigned-off-by: {}'.format(repo.name, pr2.number, reviewer),
+            'title PR2\n\nbody PR2\n\ncloses {}#{}\n\nSigned-off-by: {}\n'.format(repo.name, pr2.number, reviewer),
             p1,
             node('commit_PR2_01', node('commit_PR2_00', node('initial')))
         )
@@ -2741,8 +2743,8 @@ class TestBatching(object):
 
         staging = log_to_node(log)
         reviewer = get_partner(env, users["reviewer"]).formatted_email
-        expected = node('commit_PR2_00\n\ncloses {}#{}\n\nSigned-off-by: {}'.format(repo.name, pr2.number, reviewer),
-             node('commit_PR1_00\n\ncloses {}#{}\n\nSigned-off-by: {}'.format(repo.name, pr1.number, reviewer),
+        expected = node('commit_PR2_00\n\ncloses {}#{}\n\nSigned-off-by: {}\n'.format(repo.name, pr2.number, reviewer),
+             node('commit_PR1_00\n\ncloses {}#{}\n\nSigned-off-by: {}\n'.format(repo.name, pr1.number, reviewer),
                   node('initial')))
         assert staging == expected
 
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 57698426..908b7db4 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -435,7 +435,7 @@ def test_merge_fail(env, project, repo_a, repo_b, users, config):
         )
     env.run_crons()
 
-    s2 = to_pr(env, pr2a) | to_pr(env, pr2b)
+    pr2a_id, pr2b_id = s2 = to_pr(env, pr2a) | to_pr(env, pr2b)
     st = env['runbot_merge.stagings'].search([])
     assert set(st.batch_ids.prs.ids) == set(s2.ids)
 
@@ -453,12 +453,14 @@ def test_merge_fail(env, project, repo_a, repo_b, users, config):
         c['commit']['message']
         for c in repo_a.log('heads/staging.master')
     ] == [
-        """commit_do-b-thing_00
+        f"""\
+commit_do-b-thing_00
 
-closes %s
+closes {pr2a_id.display_name}
 
-Related: %s
-Signed-off-by: %s""" % (s2[0].display_name, s2[1].display_name, reviewer),
+Related: {pr2b_id.display_name}
+Signed-off-by: {reviewer}
+""",
         'initial'
     ], "dummy commit + squash-merged PR commit + root commit"
 

From 4d2c0f86e1f3e058f81aabad832a666739d980e3 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 23 Aug 2023 14:09:32 +0200
Subject: [PATCH 039/192] [CHG] runbot_merge: convert freeze wizard to local
 repo

Probably less necessary than for the regular staging stuff, but might
as well while at it.

Requires updating one of the test to generate a non-ff push, as
O_CREAT doesn't exist at the git level, and the client (and it is
client-side) only protects against force pushes. So there is no way to
trigger an issue with just the creation of the new branch, it needs to
exist *and point to a non-ancestor commit*.

Also remove a sleep in the ref update loop as there are no ref updates
anymore, until the very final sync via git.

NB: maybe it'd be possible to push both bump and release PRs together
for each repo, but getting which update failed in case of failure
seems difficult.
---
 runbot_merge/github.py                        | 86 -----------------
 .../models/project_freeze/__init__.py         | 94 ++++++++++---------
 runbot_merge/tests/test_multirepo.py          | 14 +--
 3 files changed, 56 insertions(+), 138 deletions(-)

diff --git a/runbot_merge/github.py b/runbot_merge/github.py
index 3fdec209..1f31e03c 100644
--- a/runbot_merge/github.py
+++ b/runbot_merge/github.py
@@ -313,92 +313,6 @@ class GH(object):
                     f"Sanity check ref update of {branch}, expected {sha} got {head}"
         return status
 
-    def merge(self, sha, dest, message):
-        r = self('post', 'merges', json={
-            'base': dest,
-            'head': sha,
-            'commit_message': message,
-        }, check={409: MergeError})
-        try:
-            r = r.json()
-        except Exception:
-            raise MergeError("got non-JSON reponse from github: %s %s (%s)" % (r.status_code, r.reason, r.text))
-        _logger.debug(
-            "merge(%s, %s (%s), %s) -> %s",
-            self._repo, dest, r['parents'][0]['sha'],
-            shorten(message), r['sha']
-        )
-        return dict(r['commit'], sha=r['sha'], parents=r['parents'])
-
-    def rebase(self, pr, dest, reset=False, commits=None):
-        """ Rebase pr's commits on top of dest, updates dest unless ``reset``
-        is set.
-
-        Returns the hash of the rebased head and a map of all PR commits (to the PR they were rebased to)
-        """
-        logger = _logger.getChild('rebase')
-        original_head = self.head(dest)
-        if commits is None:
-            commits = self.commits(pr)
-
-        logger.debug("rebasing %s, %s on %s (reset=%s, commits=%s)",
-                     self._repo, pr, dest, reset, len(commits))
-
-        if not commits:
-            raise MergeError("PR has no commits")
-        prev = original_head
-        for original in commits:
-            if len(original['parents']) != 1:
-                raise MergeError(
-                    "commits with multiple parents ({sha}) can not be rebased, "
-                    "either fix the branch to remove merges or merge without "
-                    "rebasing".format_map(
-                    original
-                ))
-            tmp_msg = 'temp rebasing PR %s (%s)' % (pr, original['sha'])
-            merged = self.merge(original['sha'], dest, tmp_msg)
-
-            # whichever parent is not original['sha'] should be what dest
-            # deref'd to, and we want to check that matches the "left parent" we
-            # expect (either original_head or the previously merged commit)
-            [base_commit] = (parent['sha'] for parent in merged['parents']
-                             if parent['sha'] != original['sha'])
-            if prev != base_commit:
-                raise MergeError(
-                    f"Inconsistent view of branch {dest} while rebasing "
-                    f"PR {pr} expected commit {prev} but the other parent of "
-                    f"merge commit {merged['sha']} is {base_commit}.\n\n"
-                    f"The branch may be getting concurrently modified."
-                )
-            prev = merged['sha']
-            original['new_tree'] = merged['tree']['sha']
-
-        prev = original_head
-        mapping = {}
-        for c in commits:
-            committer = c['commit']['committer']
-            committer.pop('date')
-            copy = self('post', 'git/commits', json={
-                'message': c['commit']['message'],
-                'tree': c['new_tree'],
-                'parents': [prev],
-                'author': c['commit']['author'],
-                'committer': committer,
-            }, check={409: MergeError}).json()
-            logger.debug('copied %s to %s (parent: %s)', c['sha'], copy['sha'], prev)
-            prev = mapping[c['sha']] = copy['sha']
-
-        if reset:
-            self.set_ref(dest, original_head)
-        else:
-            self.set_ref(dest, prev)
-
-        logger.debug('rebased %s, %s on %s (reset=%s, commits=%s) -> %s',
-                      self._repo, pr, dest, reset, len(commits),
-                      prev)
-        # prev is updated after each copy so it's the rebased PR head
-        return prev, mapping
-
     # fetch various bits of issues / prs to load them
     def pr(self, number):
         return (
diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py
index b2c35d9e..7684249d 100644
--- a/runbot_merge/models/project_freeze/__init__.py
+++ b/runbot_merge/models/project_freeze/__init__.py
@@ -1,18 +1,19 @@
-import contextlib
 import enum
 import itertools
 import json
 import logging
-import time
 from collections import Counter
+from typing import Dict
 
 from markupsafe import Markup
 
 from odoo import models, fields, api, Command
-from odoo.addons.runbot_merge.exceptions import FastForwardError
 from odoo.exceptions import UserError
 from odoo.tools import drop_view_if_exists
 
+from ... import git
+from ..pull_requests import Repository
+
 _logger = logging.getLogger(__name__)
 class FreezeWizard(models.Model):
     _name = 'runbot_merge.project.freeze'
@@ -211,50 +212,50 @@ class FreezeWizard(models.Model):
         master_name = master.name
 
         gh_sessions = {r: r.github() for r in self.project_id.repo_ids}
+        repos: Dict[Repository, git.Repo] = {
+            r: git.get_local(r, 'github').check(False)
+            for r in self.project_id.repo_ids
+        }
+        for repo, copy in repos.items():
+            copy.fetch(git.source_url(repo, 'github'), '+refs/heads/*:refs/heads/*')
 
         # prep new branch (via tmp refs) on every repo
-        rel_heads = {}
+        rel_heads: Dict[Repository, str] = {}
         # store for master heads as odds are high the bump pr(s) will be on the
         # same repo as one of the release PRs
-        prevs = {}
+        prevs: Dict[Repository, str] = {}
         for rel in self.release_pr_ids:
             repo_id = rel.repository_id
             gh = gh_sessions[repo_id]
             try:
                 prev = prevs[repo_id] = gh.head(master_name)
-            except Exception:
-                raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.")
+            except Exception as e:
+                raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.") from e
 
-            # create the tmp branch to merge the PR into
-            tmp_branch = f'tmp.{self.branch_name}'
             try:
-                gh.set_ref(tmp_branch, prev)
-            except Exception as err:
-                raise UserError(f"Unable to create branch {self.branch_name} of repository {repo_id.name}: {err}.")
+                commits = gh.commits(rel.pr_id.number)
+            except Exception as e:
+                raise UserError(f"Unable to fetch commits of release PR {rel.pr_id.display_name}.") from e
 
-            rel_heads[repo_id], _ = gh.rebase(rel.pr_id.number, tmp_branch)
-            time.sleep(1)
+            rel_heads[repo_id] = repos[repo_id].rebase(prev, commits)[0]
 
         # prep bump
-        bump_heads = {}
+        bump_heads: Dict[Repository, str] = {}
         for bump in self.bump_pr_ids:
             repo_id = bump.repository_id
             gh = gh_sessions[repo_id]
 
             try:
                 prev = prevs[repo_id] = prevs.get(repo_id) or gh.head(master_name)
-            except Exception:
-                raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.")
+            except Exception as e:
+                raise UserError(f"Unable to resolve branch {master_name} of repository {repo_id.name} to a commit.") from e
 
-            # create the tmp branch to merge the PR into
-            tmp_branch = f'tmp.{master_name}'
             try:
-                gh.set_ref(tmp_branch, prev)
-            except Exception as err:
-                raise UserError(f"Unable to create branch {master_name} of repository {repo_id.name}: {err}.")
+                commits = gh.commits(bump.pr_id.number)
+            except Exception as e:
+                raise UserError(f"Unable to fetch commits of bump PR {bump.pr_id.display_name}.") from e
 
-            bump_heads[repo_id], _ = gh.rebase(bump.pr_id.number, tmp_branch)
-            time.sleep(1)
+            bump_heads[repo_id] = repos[repo_id].rebase(prev, commits)[0]
 
         deployed = {}
         # at this point we've got a bunch of tmp branches with merged release
@@ -264,38 +265,39 @@ class FreezeWizard(models.Model):
         failure = None
         for rel in self.release_pr_ids:
             repo_id = rel.repository_id
-            # helper API currently has no API to ensure we're just creating a
-            # new branch (as cheaply as possible) so do it by hand
-            status = None
-            with contextlib.suppress(Exception):
-                status = gh_sessions[repo_id].create_ref(self.branch_name, rel_heads[repo_id])
-                deployed[rel.pr_id.id] = rel_heads[repo_id]
-                to_delete.append(repo_id)
 
-            if status != 201:
+            if repos[repo_id].push(
+                git.source_url(repo_id, 'github'),
+                f'{rel_heads[repo_id]}:refs/heads/{self.branch_name}',
+            ).returncode:
                 failure = ('create', repo_id.name, self.branch_name)
                 break
+
+            deployed[rel.pr_id.id] = rel_heads[repo_id]
+            to_delete.append(repo_id)
         else: # all release deployments succeeded
             for bump in self.bump_pr_ids:
                 repo_id = bump.repository_id
-                try:
-                    gh_sessions[repo_id].fast_forward(master_name, bump_heads[repo_id])
-                    deployed[bump.pr_id.id] = bump_heads[repo_id]
-                    to_revert.append(repo_id)
-                except FastForwardError:
+                if repos[repo_id].push(
+                    git.source_url(repo_id, 'github'),
+                    f'{bump_heads[repo_id]}:refs/heads/{master_name}'
+                ).returncode:
                     failure = ('fast-forward', repo_id.name, master_name)
                     break
 
+                deployed[bump.pr_id.id] = bump_heads[repo_id]
+                to_revert.append(repo_id)
+
         if failure:
             addendums = []
             # creating the branch failed, try to delete all previous branches
             failures = []
             for prev_id in to_revert:
-                revert = gh_sessions[prev_id]('PATCH', f'git/refs/heads/{master_name}', json={
-                    'sha': prevs[prev_id],
-                    'force': True
-                }, check=False)
-                if not revert.ok:
+                if repos[prev_id].push(
+                    '-f',
+                    git.source_url(prev_id, 'github'),
+                    f'{prevs[prev_id]}:refs/heads/{master_name}',
+                ).returncode:
                     failures.append(prev_id.name)
             if failures:
                 addendums.append(
@@ -305,8 +307,10 @@ class FreezeWizard(models.Model):
                 failures.clear()
 
             for prev_id in to_delete:
-                deletion = gh_sessions[prev_id]('DELETE', f'git/refs/heads/{self.branch_name}', check=False)
-                if not deletion.ok:
+                if repos[prev_id].push(
+                    git.source_url(prev_id, 'github'),
+                    f':refs/heads/{self.branch_name}'
+                ).returncode:
                     failures.append(prev_id.name)
             if failures:
                 addendums.append(
@@ -468,7 +472,7 @@ class OpenPRLabels(models.Model):
 
     def init(self):
         super().init()
-        drop_view_if_exists(self.env.cr, "runbot_merge_freeze_labels");
+        drop_view_if_exists(self.env.cr, "runbot_merge_freeze_labels")
         self.env.cr.execute("""
         CREATE VIEW runbot_merge_freeze_labels AS (
             SELECT DISTINCT ON (label)
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 908b7db4..59b3f932 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -5,7 +5,6 @@ source branches).
 When preparing a staging, we simply want to ensure branch-matched PRs
 are staged concurrently in all repos
 """
-import json
 import time
 import xmlrpc.client
 
@@ -1256,12 +1255,12 @@ def test_freeze_complete(env, project, repo_a, repo_b, repo_c, users, config):
 
     c_b = repo_b.commit('1.1')
     assert c_b.message.startswith('Release 1.1 (B)')
-    assert repo_b.read_tree(c_b) == {'f': '1', 'version': ''}
+    assert repo_b.read_tree(c_b) == {'f': '1', 'version': '1.1'}
     assert c_b.parents[0] == master_head_b
 
     c_c = repo_c.commit('1.1')
     assert c_c.message.startswith('Release 1.1 (C)')
-    assert repo_c.read_tree(c_c) == {'f': '2', 'version': ''}
+    assert repo_c.read_tree(c_c) == {'f': '2', 'version': '1.1'}
     assert repo_c.commit(c_c.parents[0]).parents[0] == master_head_c
 
 
@@ -1272,7 +1271,7 @@ def setup_mess(repo_a, repo_b, repo_c):
             [root, _] = r.make_commits(
                 None,
                 Commit('base', tree={'version': '', 'f': '0'}),
-                Commit('release 1.0', tree={'version': '1.0'} if r is repo_a else None),
+                Commit('release 1.0', tree={'version': '1.0'}),
                 ref='heads/1.0'
             )
             master_heads.extend(r.make_commits(root, Commit('other', tree={'f': '1'}), ref='heads/master'))
@@ -1294,7 +1293,7 @@ def setup_mess(repo_a, repo_b, repo_c):
     with repo_b:
         repo_b.make_commits(
             master_heads[1],
-            Commit('Release 1.1 (B)', tree=None),
+            Commit('Release 1.1 (B)', tree={'version': '1.1'}),
             ref='heads/release-1.1'
         )
         pr_rel_b = repo_b.make_pr(target='master', head='release-1.1')
@@ -1303,7 +1302,7 @@ def setup_mess(repo_a, repo_b, repo_c):
         pr_other = repo_c.make_pr(target='master', head='whocares')
         repo_c.make_commits(
             master_heads[2],
-            Commit('Release 1.1 (C)', tree=None),
+            Commit('Release 1.1 (C)', tree={'version': '1.1'}),
             ref='heads/release-1.1'
         )
         pr_rel_c = repo_c.make_pr(target='master', head='release-1.1')
@@ -1431,7 +1430,8 @@ def test_freeze_conflict(env, project, repo_a, repo_b, repo_c, users, config):
 
     # create conflicting branch
     with repo_c:
-        repo_c.make_ref('heads/1.1', heads[2])
+        [c] = repo_c.make_commits(heads[2], Commit("exists", tree={'version': ''}))
+        repo_c.make_ref('heads/1.1', c)
 
     # actually perform the freeze
     with pytest.raises(xmlrpc.client.Fault) as e:

From f0344fd34ac4d12b5a1b96ada52957928fe276f7 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 25 Aug 2023 11:06:04 +0200
Subject: [PATCH 040/192] [ADD] runbot_merge: link back from commit to PR

---
 runbot_merge/models/pull_requests.py | 103 +++++++++++++++------------
 runbot_merge/views/mergebot.xml      |   8 +++
 2 files changed, 65 insertions(+), 46 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index fe9ad9e6..7d871444 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1705,6 +1705,57 @@ class FeedbackTemplate(models.Model):
             _logger.exception("Failed to render template %s", self.get_external_id())
             raise
 
+
+class StagingCommits(models.Model):
+    _name = 'runbot_merge.stagings.commits'
+    _description = "Mergeable commits for stagings, always the actually merged " \
+                   "commit, never a uniquifier"
+    _log_access = False
+
+    staging_id = fields.Many2one('runbot_merge.stagings', required=True)
+    commit_id = fields.Many2one('runbot_merge.commit', index=True, required=True)
+    repository_id = fields.Many2one('runbot_merge.repository', required=True)
+
+    def _auto_init(self):
+        super()._auto_init()
+        # the same commit can be both head and tip (?)
+        tools.create_unique_index(
+            self.env.cr, self._table + "_unique",
+            self._table, ['staging_id', 'commit_id']
+        )
+        # there should be one head per staging per repository, unless one is a
+        # real head and one is a uniquifier head
+        tools.create_unique_index(
+            self.env.cr, self._table + "_unique_per_repo",
+            self._table, ['staging_id', 'repository_id'],
+        )
+
+
+class StagingHeads(models.Model):
+    _name = 'runbot_merge.stagings.heads'
+    _description = "Staging heads, may be the staging's commit or may be a " \
+                   "uniquifier (discarded on success)"
+    _log_access = False
+
+    staging_id = fields.Many2one('runbot_merge.stagings', required=True)
+    commit_id = fields.Many2one('runbot_merge.commit', index=True, required=True)
+    repository_id = fields.Many2one('runbot_merge.repository', required=True)
+
+    def _auto_init(self):
+        super()._auto_init()
+        # the same commit can be both head and tip (?)
+        tools.create_unique_index(
+            self.env.cr, self._table + "_unique",
+            self._table, ['staging_id', 'commit_id']
+        )
+        # there should be one head per staging per repository, unless one is a
+        # real head and one is a uniquifier head
+        tools.create_unique_index(
+            self.env.cr, self._table + "_unique_per_repo",
+            self._table, ['staging_id', 'repository_id'],
+        )
+
+
 class Commit(models.Model):
     """Represents a commit onto which statuses might be posted,
     independent of everything else as commits can be created by
@@ -1719,7 +1770,7 @@ class Commit(models.Model):
 
     head_ids = fields.Many2many('runbot_merge.stagings', relation='runbot_merge_stagings_heads', column2='staging_id', column1='commit_id')
     commit_ids = fields.Many2many('runbot_merge.stagings', relation='runbot_merge_stagings_commits', column2='staging_id', column1='commit_id')
-
+    pull_requests = fields.One2many('runbot_merge.pull_requests', compute='_compute_prs')
 
     def create(self, values):
         values['to_check'] = True
@@ -1769,53 +1820,13 @@ class Commit(models.Model):
         """)
         return res
 
-class StagingCommits(models.Model):
-    _name = 'runbot_merge.stagings.commits'
-    _description = "Mergeable commits for stagings, always the actually merged " \
-                   "commit, never a uniquifier"
-    _log_access = False
+    def _compute_prs(self):
+        for c in self:
+            c.pull_requests = self.env['runbot_merge.pull_requests'].search([
+                ('head', '=', self.sha),
+            ])
 
-    staging_id = fields.Many2one('runbot_merge.stagings', required=True)
-    commit_id = fields.Many2one('runbot_merge.commit', index=True, required=True)
-    repository_id = fields.Many2one('runbot_merge.repository', required=True)
 
-    def _auto_init(self):
-        super()._auto_init()
-        # the same commit can be both head and tip (?)
-        tools.create_unique_index(
-            self.env.cr, self._table + "_unique",
-            self._table, ['staging_id', 'commit_id']
-        )
-        # there should be one head per staging per repository, unless one is a
-        # real head and one is a uniquifier head
-        tools.create_unique_index(
-            self.env.cr, self._table + "_unique_per_repo",
-            self._table, ['staging_id', 'repository_id'],
-        )
-
-class StagingHeads(models.Model):
-    _name = 'runbot_merge.stagings.heads'
-    _description = "Staging heads, may be the staging's commit or may be a " \
-                   "uniquifier (discarded on success)"
-    _log_access = False
-
-    staging_id = fields.Many2one('runbot_merge.stagings', required=True)
-    commit_id = fields.Many2one('runbot_merge.commit', index=True, required=True)
-    repository_id = fields.Many2one('runbot_merge.repository', required=True)
-
-    def _auto_init(self):
-        super()._auto_init()
-        # the same commit can be both head and tip (?)
-        tools.create_unique_index(
-            self.env.cr, self._table + "_unique",
-            self._table, ['staging_id', 'commit_id']
-        )
-        # there should be one head per staging per repository, unless one is a
-        # real head and one is a uniquifier head
-        tools.create_unique_index(
-            self.env.cr, self._table + "_unique_per_repo",
-            self._table, ['staging_id', 'repository_id'],
-        )
 class Stagings(models.Model):
     _name = _description = 'runbot_merge.stagings'
 
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index a05fcc57..1650bfd4 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -291,6 +291,14 @@
                         <h1><field name="sha"/></h1>
                     </div>
                     <field name="statuses" widget="json"/>
+                    <separator string="Pull Requests"/>
+                    <field name="pull_requests" nolabel="1">
+                        <tree>
+                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                            <field name="display_name"/>
+                            <field name="state"/>
+                        </tree>
+                    </field>
                     <separator string="Stagings (commits)"/>
                     <field name="commit_ids" nolabel="1" readonly="1">
                         <tree>

From 136bb7d9fcd6f4688eaa760eb71db4068bfeb7d3 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 22 Aug 2023 08:01:38 +0200
Subject: [PATCH 041/192] [IMP] forwardport: when fetching identity avoid
 emails if possible

If the primary email is made public, it is returned directly as part
of the /users endpoint, in which case we don't need to fetch it via
/user/emails.

Also improve error messages, and fix the incorrect checks on the
existence of the github name and email. And allow manually updating
both via the project form.
---
 forwardport/data/views.xml    |  4 ++--
 forwardport/models/project.py | 21 +++++++++++++++------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/forwardport/data/views.xml b/forwardport/data/views.xml
index c64f9bb6..4536b79f 100644
--- a/forwardport/data/views.xml
+++ b/forwardport/data/views.xml
@@ -175,8 +175,8 @@
                         <field string="Token" name="fp_github_token"/>
                     </group>
                     <group>
-                        <field string="Bot Name" name="fp_github_name"/>
-                        <field string="Bot Email" name="fp_github_email"/>
+                        <field string="Bot Name" name="fp_github_name" readonly="0"/>
+                        <field string="Bot Email" name="fp_github_email" readonly="0"/>
                     </group>
                 </group>
             </xpath>
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index cf60f68a..31b75265 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -66,28 +66,37 @@ class Project(models.Model):
     def _compute_git_identity(self):
         s = requests.Session()
         for project in self:
-            if not project.fp_github_token or (self.fp_github_name and self.fp_github_email):
+            if not project.fp_github_token or (project.fp_github_name and project.fp_github_email):
                 continue
 
             r0 = s.get('https://api.github.com/user', headers={
                 'Authorization': 'token %s' % project.fp_github_token
             })
+            if not r0.ok:
+                _logger.error("Failed to fetch forward bot information for project %s: %s", project.name, r0.text or r0.content)
+                continue
+
+            user = r0.json()
+            project.fp_github_name = user['name'] or user['login']
+            if email := user['email']:
+                project.fp_github_email = email
+                continue
+
             if 'user:email' not in set(re.split(r',\s*', r0.headers['x-oauth-scopes'])):
-                raise UserError(_("The forward-port github token needs the user:email scope to fetch the bot's identity."))
+                raise UserError("The forward-port github token needs the user:email scope to fetch the bot's identity.")
             r1 = s.get('https://api.github.com/user/emails', headers={
                 'Authorization': 'token %s' % project.fp_github_token
             })
-            if not (r0.ok and r1.ok):
-                _logger.error("Failed to fetch bot information for project %s: %s", project.name, (r0.text or r0.content) if not r0.ok else (r1.text or r1.content))
+            if not r1.ok:
+                _logger.error("Failed to fetch forward bot emails for project %s: %s", project.name, r1.text or r1.content)
                 continue
-            project.fp_github_name = r0.json()['login']
             project.fp_github_email = next((
                 entry['email']
                 for entry in r1.json()
                 if entry['primary']
             ), None)
             if not project.fp_github_email:
-                raise UserError(_("The forward-port bot needs a primary email set up."))
+                raise UserError("The forward-port bot needs a public or primary email set up.")
 
     def write(self, vals):
         # check on branches both active and inactive so disabling branches doesn't

From 73e4ac6066039f5d3e5dbb8d12eef1247aa295a6 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 29 Aug 2023 13:26:12 +0200
Subject: [PATCH 042/192] [REM] runbot_merge: check_visibility

Its sole use was removed with the switch to local staging, but I
missed removing it.

Closes #625 as there is no need to update it to v2 smart protocol.
---
 runbot_merge/models/stagings_create.py | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index 544712a7..42b1aad5 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -11,7 +11,6 @@ from itertools import takewhile
 from operator import itemgetter
 from typing import Dict, Union, Optional, Literal, Callable, Iterator, Tuple, List, TypeAlias
 
-import requests
 from werkzeug.datastructures import Headers
 
 from odoo import api, models, fields
@@ -264,23 +263,6 @@ def stage_batches(branch: Branch, batched_prs: List[PullRequests], staging_state
                 )
     return staged
 
-def check_visibility(repo: Repository, branch_name: str, expected_head: str, token: str):
-    """ Checks the repository actual to see if the new / expected head is
-    now visible
-    """
-    # v1 protocol provides URL for ref discovery: https://github.com/git/git/blob/6e0cc6776106079ed4efa0cc9abace4107657abf/Documentation/technical/http-protocol.txt#L187
-    # for more complete client this is also the capabilities discovery and
-    # the "entry point" for the service
-    url = 'https://github.com/{}.git/info/refs?service=git-upload-pack'.format(repo.name)
-    with requests.get(url, stream=True, auth=(token, '')) as resp:
-        if not resp.ok:
-            return False
-        for head, ref in parse_refs_smart(resp.raw.read):
-            if ref != ('refs/heads/' + branch_name):
-                continue
-            return head == expected_head
-        return False
-
 
 refline = re.compile(rb'([\da-f]{40}) ([^\0\n]+)(\0.*)?\n?')
 ZERO_REF = b'0'*40

From 302fd42cae2cbe54778d018aa8fafa51c201dfec Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 29 Aug 2023 15:59:05 +0200
Subject: [PATCH 043/192] [ADD] forwardport: message on parent of detached PR

Currently a user is not notified that the parent of a detached PR
needs to be independently approved and may miss that information. Add
a notification to *that* PR as well.

Fixes #788
---
 forwardport/models/project.py                 | 38 ++++++++-----
 forwardport/tests/test_updates.py             | 56 ++++++++++++++++++-
 ..._merge.pull_requests.feedback.template.csv |  4 ++
 3 files changed, 82 insertions(+), 16 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index dd91d07a..498c8476 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -29,7 +29,7 @@ from pathlib import Path
 import dateutil.relativedelta
 import requests
 
-from odoo import _, models, fields, api
+from odoo import models, fields, api
 from odoo.osv import expression
 from odoo.exceptions import UserError
 from odoo.tools.misc import topological_sort, groupby
@@ -303,7 +303,11 @@ class PullRequests(models.Model):
         # also a bit odd to only handle updating 1 head at a time, but then
         # again 2 PRs with same head is weird so...
         newhead = vals.get('head')
-        with_parents = self.filtered('parent_id')
+        with_parents = {
+            p: p.parent_id
+            for p in self
+            if p.parent_id
+        }
         closed_fp = self.filtered(lambda p: p.state == 'closed' and p.source_id)
         if newhead and not self.env.context.get('ignore_head_update') and newhead != self.head:
             vals.setdefault('parent_id', False)
@@ -323,14 +327,21 @@ class PullRequests(models.Model):
             vals['merge_date'] = fields.Datetime.now()
         r = super().write(vals)
         if self.env.context.get('forwardport_detach_warn', True):
-            for p in with_parents:
-                if not p.parent_id:
-                    self.env.ref('runbot_merge.forwardport.update.detached')._send(
-                        repository=p.repository,
-                        pull_request=p.number,
-                        token_field='fp_github_token',
-                        format_args={'pr': p},
-                    )
+            for p, parent in with_parents.items():
+                if p.parent_id:
+                    continue
+                self.env.ref('runbot_merge.forwardport.update.detached')._send(
+                    repository=p.repository,
+                    pull_request=p.number,
+                    token_field='fp_github_token',
+                    format_args={'pr': p},
+                )
+                self.env.ref('runbot_merge.forwardport.update.parent')._send(
+                    repository=parent.repository,
+                    pull_request=parent.number,
+                    token_field='fp_github_token',
+                    format_args={'pr': parent, 'child': p},
+                )
         for p in closed_fp.filtered(lambda p: p.state != 'closed'):
             self.env.ref('runbot_merge.forwardport.reopen.detached')._send(
                 repository=p.repository,
@@ -927,11 +938,8 @@ class PullRequests(models.Model):
             head_commit = commits[-1]['commit']
 
             to_tuple = operator.itemgetter('name', 'email')
-            to_dict = lambda term, vals: {
-                'GIT_%s_NAME' % term: vals[0],
-                'GIT_%s_EMAIL' % term: vals[1],
-                'GIT_%s_DATE' % term: vals[2],
-            }
+            def to_dict(term, vals):
+                return {'GIT_%s_NAME' % term: vals[0], 'GIT_%s_EMAIL' % term: vals[1], 'GIT_%s_DATE' % term: vals[2]}
             authors, committers = set(), set()
             for c in (c['commit'] for c in commits):
                 authors.add(to_tuple(c['author']))
diff --git a/forwardport/tests/test_updates.py b/forwardport/tests/test_updates.py
index 1635fc5e..c83afde0 100644
--- a/forwardport/tests/test_updates.py
+++ b/forwardport/tests/test_updates.py
@@ -15,6 +15,14 @@ def test_update_pr(env, config, make_repo, users):
     only this one and its dependent should be updated?
     """
     prod, _ = make_basic(env, config, make_repo)
+    # create a branch d from c so we can have 3 forward ports PRs, not just 2,
+    # for additional checks
+    env['runbot_merge.project'].search([]).write({
+        'branch_ids': [(0, 0, {'name': 'd', 'sequence': 40})]
+    })
+    with prod:
+        prod.make_commits('c', Commit('1111', tree={'i': 'a'}), ref='heads/d')
+    
     with prod:
         [p_1] = prod.make_commits(
             'a',
@@ -37,7 +45,7 @@ def test_update_pr(env, config, make_repo, users):
     pr0_id, pr1_id = env['runbot_merge.pull_requests'].search([], order='number')
 
     fp_intermediate = (users['user'], '''\
-This PR targets b and is part of the forward-port chain. Further PRs will be created up to c.
+This PR targets b and is part of the forward-port chain. Further PRs will be created up to d.
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 ''')
@@ -121,6 +129,52 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
         'h': 'a',
         'x': '5'
     }, "the followup FP should also have the update"
+    
+    with prod:
+        prod.post_status(pr2_id.head, 'success', 'ci/runbot')
+        prod.post_status(pr2_id.head, 'success', 'legal/cla')
+    env.run_crons()
+    
+    _0, _1, _2, pr3_id = env['runbot_merge.pull_requests'].search([], order='number')
+    assert pr3_id.parent_id == pr2_id
+    # don't bother updating heads (?)
+    pr3_id.write({'parent_id': False, 'detach_reason': "testing"})
+    # pump feedback messages
+    env.run_crons()
+    
+    pr3 = prod.get_pr(pr3_id.number)
+    assert pr3.comments == [
+        seen(env, pr3, users),
+        (users['user'], f"""\
+@{users['user']} @{users['reviewer']} this PR targets d and is the last of the forward-port chain containing:
+* {pr2_id.display_name}
+
+To merge the full chain, use
+> @{pr3_id.repository.project_id.fp_github_name} r+
+
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+"""),
+        (users['user'], f"@{users['user']} @{users['reviewer']} this PR was "
+                        f"modified / updated and has become a normal PR. It "
+                        f"should be merged the normal way "
+                        f"(via @{pr3_id.repository.project_id.github_prefix})"
+        )
+    ]
+    pr2 = prod.get_pr(pr2_id.number)
+    assert pr2.comments == [
+        seen(env, pr2, users),
+        (users['user'], """\
+This PR targets c and is part of the forward-port chain. Further PRs will be created up to d.
+
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+"""),
+        (users['user'], f"@{users['user']} @{users['reviewer']} child PR "
+                        f"{pr3_id.display_name} was modified / updated and has "
+                        f"become a normal PR. This PR (and any of its parents) "
+                        f"will need to be merged independently as approvals "
+                        f"won't cross."),
+    ]
+    
 
 def test_update_merged(env, make_repo, config, users):
     """ Strange things happen when an FP gets closed / merged but then its
diff --git a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
index 70ed7397..8524d290 100644
--- a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
+++ b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
@@ -110,6 +110,10 @@ stderr: markdown-formatted stderr of git, if any"
 runbot_merge.forwardport.update.detached,{pr.ping}this PR was modified / updated and has become a normal PR. It should be merged the normal way (via @{pr.repository.project_id.github_prefix}),"Comment when a forwardport PR gets updated, documents that the PR now needs to be merged the “normal” way.
 
 pr: the pr in question "
+runbot_merge.forwardport.update.parent,{pr.ping}child PR {child.display_name} was modified / updated and has become a normal PR. This PR (and any of its parents) will need to be merged independently as approvals won't cross.,"Sent to an open PR when its direct child has been detached.
+
+pr: the pr
+child: its detached child"
 runbot_merge.forwardport.reopen.detached,{pr.ping}this PR was closed then reopened. It should be merged the normal way (via @{pr.repository.project_id.github_prefix}),"Comment when a forwardport PR gets closed then reopened, documents that the PR is now in a detached state.
 
 pr: the pr in question"

From 69f5cac2d7b3fa1df68e123c59adc6bebbdb8373 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 30 Aug 2023 11:43:13 +0200
Subject: [PATCH 044/192] [FIX] runbot_merge: support non-ascii secrets &
 sha256 signatures

Per the Github webhook documentation:

1. sha1 signatures are deprecated, github recommends sha256 (though
   that's unlikely to be a concern anyway), and dummy-central supports
   both so it should be no issue.

   > If possible, we recommend that you use the x-hub-signature-256
   > header for improved security.

2. Non-ascii secrets are supported and should be utf8-encoded to
   compute signatures... that's not actually documented as github docs
   only mention payload encoding but it seems to make sense anyway.

Also improve the warning message by replacing the signature (which is
useless) by the delivery id (which could allow introspecing the hook
or something).
---
 runbot_merge/controllers/__init__.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 3e693824..5d42abe5 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -98,10 +98,10 @@ class MergebotController(Controller):
             ('name', '=', repo),
         ]).project_id.secret
         if secret:
-            signature = 'sha1=' + hmac.new(secret.encode('ascii'), req.get_data(), hashlib.sha1).hexdigest()
-            if not hmac.compare_digest(signature, req.headers.get('X-Hub-Signature', '')):
-                _logger.warning("Ignored hook with incorrect signature %s",
-                             req.headers.get('X-Hub-Signature'))
+            signature = 'sha256=' + hmac.new(secret.encode(), req.get_data(), hashlib.sha256).hexdigest()
+            if not hmac.compare_digest(signature, req.headers.get('X-Hub-Signature-256', '')):
+                _logger.warning("Ignored hook %s with incorrect signature",
+                                req.headers.get('X-Github-Delivery'))
                 return werkzeug.exceptions.Forbidden()
 
         sentry_sdk.set_context('webhook', request.jsonrequest)

From 65ed7c51bcf42ab7a656197a53717afee5f54f99 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 30 Aug 2023 12:10:46 +0200
Subject: [PATCH 045/192] [IMP] *: note to merge using mergebot in conflict
 message

The message has a lot of info, but left the merging bit
unwritten. Correct this issue.

Fixes #765
---
 forwardport/tests/test_conflicts.py           | 28 +++++++++++++++++++
 mergebot_test_utils/utils.py                  |  2 +-
 ..._merge.pull_requests.feedback.template.csv |  2 ++
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/forwardport/tests/test_conflicts.py b/forwardport/tests/test_conflicts.py
index d5bb3486..1d29df2d 100644
--- a/forwardport/tests/test_conflicts.py
+++ b/forwardport/tests/test_conflicts.py
@@ -50,6 +50,7 @@ def test_conflict(env, config, make_repo, users):
     assert prc_id.state == 'opened'
 
     p = prod.commit(p_0)
+    prc = prod.get_pr(prc_id.number)
     c = prod.commit(prc_id.head)
     assert c.author == p.author
     # ignore date as we're specifically not keeping the original's
@@ -66,6 +67,33 @@ xxx
 >>>\x3e>>> [\da-f]{7,}.*
 '''),
     }
+    assert prc.comments == [
+        seen(env, prc, users),
+        (users['user'], re_matches(
+fr'''@{users['user']} @{users['reviewer']} cherrypicking of pull request {pra_id.display_name} failed\.
+
+stdout:
+```
+Auto-merging h
+CONFLICT \(add/add\): Merge conflict in h
+
+```
+
+stderr:
+```
+.*
+```
+
+Either perform the forward-port manually \(and push to this branch, proceeding as usual\) or close this PR \(maybe\?\)\.
+
+In the former case, you may want to edit this PR message as well\.
+
+:warning: after resolving this conflict, you will need to merge it via @{project.github_prefix}\.
+
+More info at https://github\.com/odoo/odoo/wiki/Mergebot#forward-port
+''', re.DOTALL))
+    ]
+
     prb = prod.get_pr(prb_id.number)
     assert prb.comments == [
         seen(env, prb, users),
diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index fae0d208..922eb437 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -50,7 +50,7 @@ class re_matches:
         return self._r.match(text)
 
     def __repr__(self):
-        return self._r.pattern + '...'
+        return repr(self._r.pattern)
 
 def seen(env, pr, users):
     return users['user'], f'[Pull request status dashboard]({to_pr(env, pr).url}).'
diff --git a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
index 8524d290..bbe809ff 100644
--- a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
+++ b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
@@ -143,6 +143,8 @@ runbot_merge.forwardport.failure,"{pr.ping}cherrypicking of pull request {pr.sou
 Either perform the forward-port manually (and push to this branch, proceeding as usual) or close this PR (maybe?).
 
 In the former case, you may want to edit this PR message as well.
+
+:warning: after resolving this conflict, you will need to merge it via @{pr.repository.project_id.github_prefix}.
 {footer}","Comment when a forward-port failed.
 
 pr: the new pr (in failure)

From 2c177c83f6d49254f3bc07b9e69dcfb820c3ebdf Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 30 Aug 2023 12:24:05 +0200
Subject: [PATCH 046/192] [IMP] forwardport: check fwbot approval for a
 conflicting PR

A conflicting forward port should not be approvable via fwbot, it
should be merged like a novel PR (via the mergebot).
---
 forwardport/tests/test_conflicts.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/forwardport/tests/test_conflicts.py b/forwardport/tests/test_conflicts.py
index 1d29df2d..eeeaf7ea 100644
--- a/forwardport/tests/test_conflicts.py
+++ b/forwardport/tests/test_conflicts.py
@@ -93,6 +93,10 @@ In the former case, you may want to edit this PR message as well\.
 More info at https://github\.com/odoo/odoo/wiki/Mergebot#forward-port
 ''', re.DOTALL))
     ]
+    with prod:
+        prc.post_comment(f'@{project.fp_github_name} r+', config['role_reviewer']['token'])
+    env.run_crons()
+    assert prc_id.state == 'opened', "approving via fw should not work on a conflict"
 
     prb = prod.get_pr(prb_id.number)
     assert prb.comments == [

From 2fb26f10fb854f2fed0dfafe709772ec4c5f46dc Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 31 Aug 2023 08:52:08 +0200
Subject: [PATCH 047/192] [IMP] *: make dummy saas_worker module installable

And install it. And add a hook to trigger "ready crons" (including
triggered crons).

Rather convenient to install test helpers inside the SUT.
---
 conftest.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/conftest.py b/conftest.py
index 3d936e56..3bfb9825 100644
--- a/conftest.py
+++ b/conftest.py
@@ -302,7 +302,7 @@ class DbDict(dict):
             subprocess.run([
                 'odoo', '--no-http',
                 *(['--addons-path', self._adpath] if self._adpath else []),
-                '-d', db, '-i', module + ',auth_oauth',
+                '-d', db, '-i', module + ',saas_worker,auth_oauth',
                 '--max-cron-threads', '0',
                 '--stop-after-init',
                 '--log-level', 'warn'
@@ -317,7 +317,7 @@ class DbDict(dict):
         return db
 
 @pytest.fixture(scope='session')
-def dbcache(request, tmp_path_factory):
+def dbcache(request, tmp_path_factory, addons_path):
     """ Creates template DB once per run, then just duplicates it before
     starting odoo and running the testcase
     """
@@ -327,7 +327,7 @@ def dbcache(request, tmp_path_factory):
         # level up to deref it
         shared_dir = shared_dir.parent
 
-    dbs = DbDict(request.config.getoption('--addons-path'), shared_dir)
+    dbs = DbDict(addons_path, shared_dir)
     yield dbs
 
 @pytest.fixture
@@ -381,7 +381,17 @@ def dummy_addons_path():
     with tempfile.TemporaryDirectory() as dummy_addons_path:
         mod = pathlib.Path(dummy_addons_path, 'saas_worker')
         mod.mkdir(0o700)
-        (mod / '__init__.py').write_bytes(b'')
+        (mod / '__init__.py').write_text('''\
+from odoo import api, fields, models
+
+
+class Base(models.AbstractModel):
+    _inherit = 'base'
+
+    def run_crons(self):
+        self.env['ir.cron']._process_jobs(self.env.cr.dbname)
+        return True
+''', encoding='utf-8')
         (mod / '__manifest__.py').write_text(pprint.pformat({
             'name': 'dummy saas_worker',
             'version': '1.0',
@@ -393,19 +403,21 @@ def from_role(*_, **__):
 
         yield dummy_addons_path
 
+@pytest.fixture(scope='session')
+def addons_path(request, dummy_addons_path):
+    return ','.join(map(str, filter(None, [
+        request.config.getoption('--addons-path'),
+        dummy_addons_path,
+    ])))
+
 @pytest.fixture
-def server(request, db, port, module, dummy_addons_path, tmpdir):
+def server(request, db, port, module, addons_path, tmpdir):
     log_handlers = [
         'odoo.modules.loading:WARNING',
     ]
     if not request.config.getoption('--log-github'):
         log_handlers.append('github_requests:WARNING')
 
-    addons_path = ','.join(map(str, filter(None, [
-        request.config.getoption('--addons-path'),
-        dummy_addons_path,
-    ])))
-
     cov = []
     if request.config.getoption('--coverage'):
         cov = ['coverage', 'run', '-p', '--source=odoo.addons.runbot_merge,odoo.addons.forwardport', '--branch']

From 76f4ed3bf647458bffeab30134905fae00ed3f39 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 31 Aug 2023 09:03:17 +0200
Subject: [PATCH 048/192] [ADD] runbot_merge: delete scratch branches when a
 branch is disabled

If a branch `foo` is disabled, then `tmp.foo` and `staging.foo` become
unnecessary (with #247 fixed the tmp refs are not used for creating
stagings anymore, but for now they're still used for the "safety
dance" of merging a successful staging into the corresponding
mainline).

Fixes #605
---
 runbot_merge/__manifest__.py                  |  1 +
 runbot_merge/models/crons/__init__.py         |  1 +
 .../models/crons/cleanup_scratch_branches.py  | 32 +++++++++++++++++++
 .../models/crons/cleanup_scratch_branches.xml | 23 +++++++++++++
 runbot_merge/models/pull_requests.py          |  1 +
 runbot_merge/tests/test_disabled_branch.py    | 18 +++++++++++
 6 files changed, 76 insertions(+)
 create mode 100644 runbot_merge/models/crons/cleanup_scratch_branches.py
 create mode 100644 runbot_merge/models/crons/cleanup_scratch_branches.xml

diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py
index 1f6ae7f4..266605d3 100644
--- a/runbot_merge/__manifest__.py
+++ b/runbot_merge/__manifest__.py
@@ -8,6 +8,7 @@
 
         'data/merge_cron.xml',
         'models/crons/git_maintenance.xml',
+        'models/crons/cleanup_scratch_branches.xml',
         'data/runbot_merge.pull_requests.feedback.template.csv',
         'views/res_partner.xml',
         'views/runbot_merge_project.xml',
diff --git a/runbot_merge/models/crons/__init__.py b/runbot_merge/models/crons/__init__.py
index 939a44cc..f3eed923 100644
--- a/runbot_merge/models/crons/__init__.py
+++ b/runbot_merge/models/crons/__init__.py
@@ -1 +1,2 @@
 from . import git_maintenance
+from . import cleanup_scratch_branches
diff --git a/runbot_merge/models/crons/cleanup_scratch_branches.py b/runbot_merge/models/crons/cleanup_scratch_branches.py
new file mode 100644
index 00000000..1e7b6274
--- /dev/null
+++ b/runbot_merge/models/crons/cleanup_scratch_branches.py
@@ -0,0 +1,32 @@
+import logging
+
+from odoo import models
+
+
+_logger = logging.getLogger(__name__)
+class BranchCleanup(models.TransientModel):
+    _name = 'runbot_merge.branch_cleanup'
+    _description = "cleans up scratch refs for deactivated branches"
+
+    def _run(self):
+        deactivated = self.env['runbot_merge.branch'].search([
+            ('active', '=', False),
+            ('write_date', '>=', self.env.context['lastcall']),
+        ])
+        _logger.info(
+            "deleting scratch (tmp and staging) refs for branches %s",
+            ', '.join(b.name for b in deactivated)
+        )
+        # loop around the repos first, so we can reuse the gh instance
+        for r in deactivated.mapped('project_id.repo_ids'):
+            gh = r.github()
+            for b in deactivated:
+                if b.project_id != r.project_id:
+                    continue
+
+                res = gh('delete', f'git/refs/heads/tmp.{b.name}', check=False)
+                if res.status_code != 204:
+                    _logger.info("no tmp branch found for %s:%s", r.name, b.name)
+                res = gh('delete', f'git/refs/heads/staging.{b.name}', check=False)
+                if res.status_code != 204:
+                    _logger.info("no staging branch found for %s:%s", res.name, b.name)
diff --git a/runbot_merge/models/crons/cleanup_scratch_branches.xml b/runbot_merge/models/crons/cleanup_scratch_branches.xml
new file mode 100644
index 00000000..e9172f6d
--- /dev/null
+++ b/runbot_merge/models/crons/cleanup_scratch_branches.xml
@@ -0,0 +1,23 @@
+<odoo>
+    <record id="access_branch_cleanup" model="ir.model.access">
+        <field name="name">Access to branch cleanup is useless</field>
+        <field name="model_id" ref="model_runbot_merge_branch_cleanup"/>
+        <field name="perm_read">0</field>
+        <field name="perm_create">0</field>
+        <field name="perm_write">0</field>
+        <field name="perm_unlink">0</field>
+    </record>
+
+    <record model="ir.cron" id="branch_cleanup">
+        <field name="name">Removal of scratch refs for deactivated branch</field>
+        <field name="model_id" ref="model_runbot_merge_branch_cleanup"/>
+        <field name="state">code</field>
+        <field name="code">model._run()</field>
+        <!--
+        nota: even though this is only triggered, numbercall has to be
+        non-zero because the counter is taken in account by cron triggers
+        -->
+        <field name="numbercall">-1</field>
+        <field name="doall" eval="False"/>
+    </record>
+</odoo>
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index a6d210bf..96581736 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -282,6 +282,7 @@ class Branch(models.Model):
                 'pull_request': pr.number,
                 'message': tmpl._format(pr=pr),
             } for pr in self.prs])
+            self.env.ref('runbot_merge.branch_cleanup')._trigger()
         return True
 
     @api.depends('staging_ids.active')
diff --git a/runbot_merge/tests/test_disabled_branch.py b/runbot_merge/tests/test_disabled_branch.py
index 17282985..08cbf117 100644
--- a/runbot_merge/tests/test_disabled_branch.py
+++ b/runbot_merge/tests/test_disabled_branch.py
@@ -1,9 +1,15 @@
+import pytest
+
 from utils import seen, Commit, pr_page
 
 def test_existing_pr_disabled_branch(env, project, make_repo, setreviewers, config, users, page):
     """ PRs to disabled branches are ignored, but what if the PR exists *before*
     the branch is disabled?
     """
+    # run crons from template to clean up the queue before possibly creating
+    # new work
+    assert env['base'].run_crons()
+
     repo = make_repo('repo')
     project.branch_ids.sequence = 0
     project.write({'branch_ids': [
@@ -81,6 +87,18 @@ def test_existing_pr_disabled_branch(env, project, make_repo, setreviewers, conf
     assert pr_id.target == env['runbot_merge.branch'].search([('name', '=', 'other2')])
     assert pr_id.staging_id
 
+    # staging of `pr` should have generated a staging branch
+    _ = repo.get_ref('heads/staging.other')
+    # stagings should not need a tmp branch anymore, so this should not exist
+    with pytest.raises(AssertionError, match=r'Not Found'):
+        repo.get_ref('heads/tmp.other')
+
+    assert env['base'].run_crons()
+
+    # triggered cleanup should have deleted the staging for the disabled `other`
+    # target branch
+    with pytest.raises(AssertionError, match=r'Not Found'):
+        repo.get_ref('heads/staging.other')
 
 def test_new_pr_no_branch(env, project, make_repo, setreviewers, users):
     """ A new PR to an *unknown* branch should be ignored and warn

From ea2857ec31a9821e3a434bc1f68ddbbc82cb5e04 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 31 Aug 2023 12:31:59 +0200
Subject: [PATCH 049/192] [IMP] forwardport: parametrize main fw limits

Probably a bit more expensive, but makes the code shorter and more
straightforward.
---
 forwardport/tests/test_limit.py | 58 +++++++++++++--------------------
 1 file changed, 22 insertions(+), 36 deletions(-)

diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index ecdfd585..ed9ff35a 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -1,49 +1,35 @@
-import collections
+import pytest
 
-from utils import seen, Commit, make_basic
+from utils import seen, Commit, make_basic, to_pr
 
-Description = collections.namedtuple('Restriction', 'source limit')
-def test_configure(env, config, make_repo):
-    """ Checks that configuring an FP limit on a PR is respected
 
-    * limits to not the latest
-    * limits to the current target (= no FP)
-    * limits to an earlier branch (???)
-    """
+@pytest.mark.parametrize('source,limit,count', [
+    pytest.param('a', 'b', 1, id='not-last'),
+    pytest.param('b', 'b', 0, id='current'),
+    pytest.param('b', 'a', 0, id='earlier'),
+])
+def test_configure_fp_limit(env, config, make_repo, source, limit, count):
     prod, other = make_basic(env, config, make_repo)
     bot_name = env['runbot_merge.project'].search([]).fp_github_name
-    descriptions = [
-        Description(source='a', limit='b'),
-        Description(source='b', limit='b'),
-        Description(source='b', limit='a'),
-    ]
-    originals = []
     with prod:
-        for i, descr in enumerate(descriptions):
-            [c] = prod.make_commits(
-                descr.source, Commit('c %d' % i, tree={str(i): str(i)}),
-                ref='heads/branch%d' % i,
-            )
-            pr = prod.make_pr(target=descr.source, head='branch%d'%i)
-            prod.post_status(c, 'success', 'legal/cla')
-            prod.post_status(c, 'success', 'ci/runbot')
-            pr.post_comment('hansen r+\n%s up to %s' % (bot_name, descr.limit), config['role_reviewer']['token'])
-            originals.append(pr.number)
+        [c] = prod.make_commits(
+            source, Commit('c', tree={'f': 'g'}),
+            ref='heads/branch',
+        )
+        pr = prod.make_pr(target=source, head='branch')
+        prod.post_status(c, 'success', 'legal/cla')
+        prod.post_status(c, 'success', 'ci/runbot')
+        pr.post_comment(f'hansen r+\n{bot_name} up to {limit}', config['role_reviewer']['token'])
     env.run_crons()
     with prod:
-        prod.post_status('staging.a', 'success', 'legal/cla')
-        prod.post_status('staging.a', 'success', 'ci/runbot')
-        prod.post_status('staging.b', 'success', 'legal/cla')
-        prod.post_status('staging.b', 'success', 'ci/runbot')
+        prod.post_status(f'staging.{source}', 'success', 'legal/cla')
+        prod.post_status(f'staging.{source}', 'success', 'ci/runbot')
     env.run_crons()
 
-    # should have created a single FP PR for 0, none for 1 and none for 2
-    prs = env['runbot_merge.pull_requests'].search([], order='number')
-    assert len(prs) == 4
-    assert prs[-1].parent_id == prs[0]
-    assert prs[0].number == originals[0]
-    assert prs[1].number == originals[1]
-    assert prs[2].number == originals[2]
+    descendants = env['runbot_merge.pull_requests'].search([
+        ('source_id', '=', to_pr(env, pr).id)
+    ])
+    assert len(descendants) == count
 
 
 def test_ignore(env, config, make_repo):

From f44b0c018ee5a2b07af8b30bd270b8bd167e006f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 6 Oct 2023 13:19:01 +0200
Subject: [PATCH 050/192] [IMP] forwardport: allow updating the fw limit after
 merging the source

Currently, once a source PR has been merged it's not possible to set
or update a limit, which can be inconvenient (e.g. people might have
forgotten to set it, or realise after the fact that setting one is not
useful, or might realise after the fact that they should *unset* it).

This PR relaxes that constraint (which is not just a relaxation as it
requires a bunch of additional work and validation), it should now be
possible to:

- update the limit to any target, as long as that target is at or
  above the current forwardport tip's
  - with the exception of a tip being forward ported, as that can't be
    cancelled
- resume a forward port stopped by a previously set limit (just
  increase it to whatever)
- set a limit from any forward-port PR
- set a limit from a source, post-merge

The process should also provide pretty chatty feedback:

- feedback on the source and closest root
- feedback about adjustments if any (e.g. setting the limit to B but
  there's already a forward port to C, the 'bot should set the limit
  to C and tell you that it happened and why)

Fixes #506
---
 forwardport/models/forwardport.py             |  22 +-
 forwardport/models/project.py                 | 197 +++++++++---
 forwardport/tests/test_limit.py               | 294 ++++++++++++++++--
 .../changelog/2023-10/free-the-limit.md       |   8 +
 4 files changed, 450 insertions(+), 71 deletions(-)
 create mode 100644 runbot_merge/changelog/2023-10/free-the-limit.md

diff --git a/forwardport/models/forwardport.py b/forwardport/models/forwardport.py
index 3bddf74d..7530d764 100644
--- a/forwardport/models/forwardport.py
+++ b/forwardport/models/forwardport.py
@@ -24,7 +24,20 @@ class Queue:
         raise NotImplementedError
 
     def _process(self):
-        for b in self.search(self._search_domain(), order='create_date, id', limit=self.limit):
+        skip = 0
+        from_clause, where_clause, params = self._search(self._search_domain(), order='create_date, id', limit=1).get_sql()
+        for _ in range(self.limit):
+            self.env.cr.execute(f"""
+                SELECT id FROM {from_clause}
+                WHERE {where_clause or "true"}
+                ORDER BY create_date, id
+                LIMIT 1 OFFSET %s
+                FOR UPDATE SKIP LOCKED
+            """, [*params, skip])
+            b = self.browse(self.env.cr.fetchone())
+            if not b:
+                return
+
             try:
                 with sentry_sdk.start_span(description=self._name):
                     b._process_item()
@@ -33,11 +46,12 @@ class Queue:
             except Exception:
                 _logger.exception("Error while processing %s, skipping", b)
                 self.env.cr.rollback()
-                b._on_failure()
+                if b._on_failure():
+                    skip += 1
                 self.env.cr.commit()
 
     def _on_failure(self):
-        pass
+        return True
 
     def _search_domain(self):
         return []
@@ -48,7 +62,7 @@ class ForwardPortTasks(models.Model, Queue):
 
     limit = 10
 
-    batch_id = fields.Many2one('runbot_merge.batch', required=True)
+    batch_id = fields.Many2one('runbot_merge.batch', required=True, index=True)
     source = fields.Selection([
         ('merge', 'Merge'),
         ('fp', 'Forward Port Followup'),
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 498c8476..adde4192 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -24,21 +24,26 @@ import re
 import subprocess
 import tempfile
 import typing
+from functools import reduce
+from operator import itemgetter
 from pathlib import Path
 
 import dateutil.relativedelta
+import psycopg2.errors
 import requests
 
 from odoo import models, fields, api
 from odoo.osv import expression
 from odoo.exceptions import UserError
-from odoo.tools.misc import topological_sort, groupby
+from odoo.tools.misc import topological_sort, groupby, Reverse
 from odoo.tools.sql import reverse_order
 from odoo.tools.appdirs import user_cache_dir
+from odoo.addons.base.models.res_partner import Partner
 from odoo.addons.runbot_merge import git, utils
-from odoo.addons.runbot_merge.models.pull_requests import RPLUS
+from odoo.addons.runbot_merge.models.pull_requests import RPLUS, Branch
 from odoo.addons.runbot_merge.models.stagings_create import Message
 
+
 footer = '\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n'
 
 DEFAULT_DELTA = dateutil.relativedelta.relativedelta(days=3)
@@ -48,6 +53,8 @@ _logger = logging.getLogger('odoo.addons.forwardport')
 class Project(models.Model):
     _inherit = 'runbot_merge.project'
 
+    id: int
+    github_prefix: str
     fp_github_token = fields.Char()
     fp_github_name = fields.Char(store=True, compute="_compute_git_identity")
     fp_github_email = fields.Char(store=True, compute="_compute_git_identity")
@@ -217,11 +224,25 @@ class Project(models.Model):
 
 class Repository(models.Model):
     _inherit = 'runbot_merge.repository'
+
+    id: int
+    project_id: Project
+    name: str
+    branch_filter: str
     fp_remote_target = fields.Char(help="where FP branches get pushed")
 
 class PullRequests(models.Model):
     _inherit = 'runbot_merge.pull_requests'
 
+    id: int
+    display_name: str
+    number: int
+    repository: Repository
+    target: Branch
+    reviewed_by: Partner
+    head: str
+    state: str
+
     statuses = fields.Text(recursive=True)
 
     limit_id = fields.Many2one('runbot_merge.branch', help="Up to which branch should this PR be forward-ported")
@@ -230,6 +251,7 @@ class PullRequests(models.Model):
         'runbot_merge.pull_requests', index=True,
         help="a PR with a parent is an automatic forward port"
     )
+    root_id = fields.Many2one('runbot_merge.pull_requests', compute='_compute_root', recursive=True)
     source_id = fields.Many2one('runbot_merge.pull_requests', index=True, help="the original source of this FP even if parents were detached along the way")
     forwardport_ids = fields.One2many('runbot_merge.pull_requests', 'source_id')
     reminder_backoff_factor = fields.Integer(default=-4, group_operator=None)
@@ -273,6 +295,10 @@ class PullRequests(models.Model):
             )
             pr.ping = s and (s + ' ')
 
+    @api.depends('parent_id.root_id')
+    def _compute_root(self):
+        for p in self:
+            p.root_id = reduce(lambda _, p: p, self._iter_ancestors())
 
     @api.model_create_single
     def create(self, vals):
@@ -291,7 +317,7 @@ class PullRequests(models.Model):
                 ast.literal_eval(repo.branch_filter or '[]')
             )[-1].id
         if vals.get('parent_id') and 'source_id' not in vals:
-            vals['source_id'] = self.browse(vals['parent_id'])._get_root().id
+            vals['source_id'] = self.browse(vals['parent_id']).root_id.id
         if vals.get('state') == 'merged':
             vals['merge_date'] = fields.Datetime.now()
         return super().create(vals)
@@ -317,12 +343,12 @@ class PullRequests(models.Model):
             # updating children
             if self.search_count([('parent_id', '=', self.id)]):
                 self.env['forwardport.updates'].create({
-                    'original_root': self._get_root().id,
+                    'original_root': self.root_id.id,
                     'new_root': self.id
                 })
 
         if vals.get('parent_id') and 'source_id' not in vals:
-            vals['source_id'] = self.browse(vals['parent_id'])._get_root().id
+            vals['source_id'] = self.browse(vals['parent_id']).root_id.id
         if vals.get('state') == 'merged':
             vals['merge_date'] = fields.Datetime.now()
         r = super().write(vals)
@@ -444,28 +470,7 @@ class PullRequests(models.Model):
                 elif not limit:
                     msg = "please provide a branch to forward-port to."
                 else:
-                    limit_id = self.env['runbot_merge.branch'].with_context(active_test=False).search([
-                        ('project_id', '=', self.repository.project_id.id),
-                        ('name', '=', limit),
-                    ])
-                    if self.source_id:
-                        msg = "forward-port limit can only be set on " \
-                              f"an origin PR ({self.source_id.display_name} " \
-                              "here) before it's merged and forward-ported."
-                    elif self.state in ['merged', 'closed']:
-                        msg = "forward-port limit can only be set before the PR is merged."
-                    elif not limit_id:
-                        msg = "there is no branch %r, it can't be used as a forward port target." % limit
-                    elif limit_id == self.target:
-                        ping = False
-                        msg = "Forward-port disabled."
-                        self.limit_id = limit_id
-                    elif not limit_id.active:
-                        msg = "branch %r is disabled, it can't be used as a forward port target." % limit_id.name
-                    else:
-                        ping = False
-                        msg = "Forward-porting to %r." % limit_id.name
-                        self.limit_id = limit_id
+                    ping, msg = self._maybe_update_limit(limit)
 
             if msg or close:
                 if msg:
@@ -480,6 +485,98 @@ class PullRequests(models.Model):
                     'token_field': 'fp_github_token',
                 })
 
+    def _maybe_update_limit(self, limit: str) -> typing.Tuple[bool, str]:
+        limit_id = self.env['runbot_merge.branch'].with_context(active_test=False).search([
+            ('project_id', '=', self.repository.project_id.id),
+            ('name', '=', limit),
+        ])
+        if not limit_id:
+            return True, f"there is no branch {limit!r}, it can't be used as a forward port target."
+
+        if limit_id != self.target and not limit_id.active:
+            return True, f"branch {limit_id.name!r} is disabled, it can't be used as a forward port target."
+
+        # not forward ported yet, just acknowledge the request
+        if not self.source_id and self.state != 'merged':
+            self.limit_id = limit_id
+            if branch_key(limit_id) <= branch_key(self.target):
+                return False, "Forward-port disabled."
+            else:
+                return False, f"Forward-porting to {limit_id.name!r}."
+
+        # if the PR has been forwardported
+        prs = (self | self.forwardport_ids | self.source_id | self.source_id.forwardport_ids)
+        tip = max(prs, key=pr_key)
+        # if the fp tip was closed it's fine
+        if tip.state == 'closed':
+            return True, f"{tip.display_name} is closed, no forward porting is going on"
+
+        prs.limit_id = limit_id
+
+        real_limit = max(limit_id, tip.target, key=branch_key)
+
+        addendum = ''
+        # check if tip was queued for forward porting, try to cancel if we're
+        # supposed to stop here
+        if real_limit == tip.target and (task := self.env['forwardport.batches'].search([('batch_id', 'in', tip.batch_ids.ids)])):
+            try:
+                with self.env.cr.savepoint():
+                    self.env.cr.execute(
+                        "SELECT FROM forwardport_batches "
+                        "WHERE id = %s FOR UPDATE NOWAIT",
+                        [task.id])
+            except psycopg2.errors.LockNotAvailable:
+                # row locked = port occurring and probably going to succeed,
+                # so next(real_limit) likely a done deal already
+                return True, (
+                    f"Forward port of {tip.display_name} likely already "
+                    f"ongoing, unable to cancel, close next forward port "
+                    f"when it completes.")
+            else:
+                self.env.cr.execute("DELETE FROM forwardport_batches WHERE id = %s", [task.id])
+
+        if real_limit != tip.target:
+            # forward porting was previously stopped at tip, and we want it to
+            # resume
+            if tip.state == 'merged':
+                self.env['forwardport.batches'].create({
+                    'batch_id': tip.batch_ids.sorted('id')[-1].id,
+                    'source': 'fp' if tip.parent_id else 'merge',
+                })
+                resumed = tip
+            else:
+                # reactivate batch
+                tip.batch_ids.sorted('id')[-1].active = True
+                resumed = tip._schedule_fp_followup()
+            if resumed:
+                addendum += f', resuming forward-port stopped at {tip.display_name}'
+
+        if real_limit != limit_id:
+            addendum += f' (instead of the requested {limit_id.name!r} because {tip.display_name} already exists)'
+
+        # get a "stable" root rather than self's to avoid divertences between
+        # PRs across a root divide (where one post-root would point to the root,
+        # and one pre-root would point to the source, or a previous root)
+        root = tip.root_id
+        # reference the root being forward ported unless we are the root
+        root_ref = '' if root == self else f' {root.display_name}'
+        msg = f"Forward-porting{root_ref} to {real_limit.name!r}{addendum}."
+        # send a message to the source & root except for self, if they exist
+        root_msg = f'Forward-porting to {real_limit.name!r} (from {self.display_name}).'
+        self.env['runbot_merge.pull_requests.feedback'].create([
+            {
+                'repository': p.repository.id,
+                'pull_request': p.number,
+                'message': root_msg,
+                'token_field': 'fp_github_token',
+            }
+            # send messages to source and root unless root is self (as it
+            # already gets the normal message)
+            for p in (self.source_id | root) - self
+        ])
+
+        return False, msg
+
     def _notify_ci_failed(self, ci):
         # only care about FP PRs which are not staged / merged yet
         # NB: probably ignore approved PRs as normal message will handle them?
@@ -501,6 +598,7 @@ class PullRequests(models.Model):
     def _schedule_fp_followup(self):
         _logger = logging.getLogger(__name__).getChild('forwardport.next')
         # if the PR has a parent and is CI-validated, enqueue the next PR
+        scheduled = self.browse(())
         for pr in self:
             _logger.info('Checking if forward-port %s (%s)', pr.display_name, pr)
             if not pr.parent_id:
@@ -548,6 +646,8 @@ class PullRequests(models.Model):
                 'batch_id': batch.id,
                 'source': 'fp',
             })
+            scheduled |= pr
+        return scheduled
 
     def _find_next_target(self, reference):
         """ Finds the branch between target and limit_id which follows
@@ -600,14 +700,15 @@ class PullRequests(models.Model):
         }
         return sorted(commits, key=lambda c: idx[c['sha']])
 
+    def _iter_ancestors(self):
+        while self:
+            yield self
+            self = self.parent_id
+
     def _iter_descendants(self):
         pr = self
-        while True:
-            pr = self.search([('parent_id', '=', pr.id)])
-            if pr:
-                yield pr
-            else:
-                break
+        while pr := self.search([('parent_id', '=', pr.id)]):
+            yield pr
 
     @api.depends('parent_id.statuses')
     def _compute_statuses(self):
@@ -619,17 +720,6 @@ class PullRequests(models.Model):
         p.update(super()._get_overrides())
         return p
 
-    def _iter_ancestors(self):
-        while self:
-            yield self
-            self = self.parent_id
-
-    def _get_root(self):
-        root = self
-        while root.parent_id:
-            root = root.parent_id
-        return root
-
     def _port_forward(self):
         if not self:
             return
@@ -720,7 +810,7 @@ class PullRequests(models.Model):
         for pr in self:
             owner, _ = pr.repository.fp_remote_target.split('/', 1)
             source = pr.source_id or pr
-            root = pr._get_root()
+            root = pr.root_id
 
             message = source.message + '\n\n' + '\n'.join(
                 "Forward-Port-Of: %s" % p.display_name
@@ -872,7 +962,7 @@ class PullRequests(models.Model):
         :rtype: (None | (str, str, str, list[commit]), Repo)
         """
         logger = _logger.getChild(str(self.id))
-        root = self._get_root()
+        root = self.root_id
         logger.info(
             "Forward-porting %s (%s) to %s",
             self.display_name, root.display_name, target_branch.name
@@ -1063,7 +1153,7 @@ stderr:
 
         # ensures all reviewers in the review path are on the PR in order:
         # original reviewer, then last conflict reviewer, then current PR
-        reviewers = (self | self._get_root() | self.source_id)\
+        reviewers = (self | self.root_id | self.source_id)\
             .mapped('reviewed_by.formatted_email')
 
         sobs = msg.headers.getlist('signed-off-by')
@@ -1127,6 +1217,19 @@ stderr:
                 }
             )
 
+
+# ordering is a bit unintuitive because the lowest sequence (and name)
+# is the last link of the fp chain, reasoning is a bit more natural the
+# other way around (highest object is the last), especially with Python
+# not really having lazy sorts in the stdlib
+def branch_key(b: Branch, /, _key=itemgetter('sequence', 'name')):
+    return Reverse(_key(b))
+
+
+def pr_key(p: PullRequests, /):
+    return branch_key(p.target)
+
+
 class Stagings(models.Model):
     _inherit = 'runbot_merge.stagings'
 
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index ed9ff35a..c96fa853 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -1,3 +1,4 @@
+
 import pytest
 
 from utils import seen, Commit, make_basic, to_pr
@@ -120,13 +121,11 @@ def test_disable(env, config, make_repo, users):
         seen(env, pr, users),
     }
 
+
 def test_limit_after_merge(env, config, make_repo, users):
-    """ If attempting to set a limit (<up to>) on a PR which is merged
-    (already forward-ported or not), or is a forward-port PR, fwbot should
-    just feedback that it won't do it
-    """
     prod, other = make_basic(env, config, make_repo)
     reviewer = config['role_reviewer']['token']
+    branch_b = env['runbot_merge.branch'].search([('name', '=', 'b')])
     branch_c = env['runbot_merge.branch'].search([('name', '=', 'c')])
     bot_name = env['runbot_merge.project'].search([]).fp_github_name
     with prod:
@@ -150,13 +149,13 @@ def test_limit_after_merge(env, config, make_repo, users):
         pr2.post_comment(bot_name + ' up to b', reviewer)
     env.run_crons()
 
-    assert p1.limit_id == p2.limit_id == branch_c, \
-        "check that limit was not updated"
+    assert p1.limit_id == p2.limit_id == branch_b
     assert pr1.comments == [
         (users['reviewer'], "hansen r+"),
         seen(env, pr1, users),
-        (users['reviewer'], bot_name + ' up to b'),
-        (users['user'], "@%s forward-port limit can only be set before the PR is merged." % users['reviewer']),
+        (users['reviewer'], f'{bot_name} up to b'),
+        (users['user'], "Forward-porting to 'b'."),
+        (users['user'], f"Forward-porting to 'b' (from {p2.display_name})."),
     ]
     assert pr2.comments == [
         seen(env, pr2, users),
@@ -165,12 +164,8 @@ This PR targets b and is part of the forward-port chain. Further PRs will be cre
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 """),
-        (users['reviewer'], bot_name + ' up to b'),
-        (users['user'], "@%s forward-port limit can only be set on an origin PR"
-                   " (%s here) before it's merged and forward-ported." % (
-            users['reviewer'],
-            p1.display_name,
-        )),
+        (users['reviewer'], f'{bot_name} up to b'),
+        (users['user'], f"Forward-porting {p1.display_name} to 'b'."),
     ]
 
     # update pr2 to detach it from pr1
@@ -186,7 +181,7 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
     assert p2.source_id == p1
 
     with prod:
-        pr2.post_comment(bot_name + ' up to b', reviewer)
+        pr2.post_comment(f'{bot_name} up to c', reviewer)
     env.run_crons()
 
     assert pr2.comments[4:] == [
@@ -195,9 +190,268 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
             users['user'], users['reviewer'],
             p2.repository.project_id.github_prefix
         )),
-        (users['reviewer'], bot_name + ' up to b'),
-        (users['user'], f"@{users['reviewer']} forward-port limit can only be "
-                        f"set on an origin PR ({p1.display_name} here) before "
-                        f"it's merged and forward-ported."
-        ),
+        (users['reviewer'], f'{bot_name} up to c'),
+        (users['user'], "Forward-porting to 'c'."),
     ]
+    with prod:
+        prod.post_status(p2.head, 'success', 'legal/cla')
+        prod.post_status(p2.head, 'success', 'ci/runbot')
+        pr2.post_comment('hansen r+', reviewer)
+    env.run_crons()
+    with prod:
+        prod.post_status('staging.b', 'success', 'legal/cla')
+        prod.post_status('staging.b', 'success', 'ci/runbot')
+    env.run_crons()
+
+    _, _, p3 = env['runbot_merge.pull_requests'].search([], order='number')
+    assert p3
+    pr3 = prod.get_pr(p3.number)
+    with prod:
+        pr3.post_comment(f"{bot_name} up to c", reviewer)
+    env.run_crons()
+    assert pr3.comments == [
+        seen(env, pr3, users),
+        (users['user'], f"""\
+@{users['user']} @{users['reviewer']} this PR targets c and is the last of the forward-port chain.
+
+To merge the full chain, use
+> @{p1.repository.project_id.fp_github_name} r+
+
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+"""),
+        (users['reviewer'], f"{bot_name} up to c"),
+        (users['user'], f"Forward-porting {p2.display_name} to 'c'."),
+    ]
+    # 7 of previous check, plus r+
+    assert pr2.comments[8:] == [
+        (users['user'], f"Forward-porting to 'c' (from {p3.display_name}).")
+    ]
+
+
+
+@pytest.mark.parametrize("update_from", [
+    pytest.param(lambda source: [('id', '=', source)], id='source'),
+    pytest.param(lambda source: [('source_id', '=', source), ('target', '=', '2')], id='child'),
+    pytest.param(lambda source: [('source_id', '=', source), ('target', '=', '3')], id='root'),
+    pytest.param(lambda source: [('source_id', '=', source), ('target', '=', '4')], id='parent'),
+    pytest.param(lambda source: [('source_id', '=', source), ('target', '=', '5')], id='current'),
+    # pytest.param(id='tip'), # doesn't exist
+])
+@pytest.mark.parametrize("limit", range(1, 6+1))
+def test_post_merge(
+        env, post_merge, users, config, branches,
+        update_from: callable,
+        limit: int,
+):
+    PRs = env['runbot_merge.pull_requests']
+    project, prod, _ = post_merge
+    reviewer = config['role_reviewer']['token']
+
+    # fetch source PR
+    [source] = PRs.search([('source_id', '=', False)])
+
+    # validate the forward ports for "child", "root", and "parent" so "current"
+    # exists and we have one more target
+    for branch in map(str, range(2, 4+1)):
+        setci(source=source, repo=prod, target=branch)
+        env.run_crons()
+    # update 3 to make it into a root
+    root = PRs.search([('source_id', '=', source.id), ('target.name', '=', '3')])
+    root.write({'parent_id': False, 'detach_reason': 'testing'})
+    # send detach messages so they're not part of the limit stuff batch
+    env.run_crons()
+
+    # cheat: we know PR numbers are assigned sequentially
+    prs = list(map(prod.get_pr, range(1, 6)))
+    before = {p.number: len(p.comments) for p in prs}
+
+    from_id = PRs.search(update_from(source.id))
+    from_ = prod.get_pr(from_id.number)
+    with prod:
+        from_.post_comment(f'{project.fp_github_name} up to {limit}', reviewer)
+    env.run_crons()
+
+    # there should always be a comment on the source and root indicating how
+    # far we port
+    # the PR we post on should have a comment indicating the correction
+    current_id = PRs.search([('number', '=', '5')])
+    actual_limit = max(limit, 5)
+    for p in prs:
+        # case for the PR on which we posted the comment
+        if p.number == from_.number:
+            root_opt = '' if p.number == root.number else f' {root.display_name}'
+            trailer = '' if actual_limit == limit else f" (instead of the requested '{limit}' because {current_id.display_name} already exists)"
+            assert p.comments[before[p.number] + 1:] == [
+                (users['user'], f"Forward-porting{root_opt} to '{actual_limit}'{trailer}.")
+            ]
+        # case for reference PRs source and root (which get their own notifications)
+        elif p.number in (source.number, root.number):
+            assert p.comments[before[p.number]:] == [
+                (users['user'], f"Forward-porting to '{actual_limit}' (from {from_id.display_name}).")
+            ]
+
+@pytest.mark.parametrize('mode', [
+    None,
+    # last forward port should fail ci, and only be validated after target bump
+    'failbump',
+    # last forward port should fail ci, then be validated, then target bump
+    'failsucceed',
+    # last forward port should be merged before bump
+    'mergetip',
+    # every forward port should be merged before bump
+    'mergeall',
+])
+def test_resume_fw(env, post_merge, users, config, branches, mode):
+    """Singleton version of test_post_merge: completes the forward porting
+    including validation then tries to increase the limit, which should resume
+    forward porting
+    """
+
+    PRs = env['runbot_merge.pull_requests']
+    project, prod, _ = post_merge
+    reviewer = config['role_reviewer']['token']
+
+    # fetch source PR
+    [source] = PRs.search([('source_id', '=', False)])
+    with prod:
+        prod.get_pr(source.number).post_comment(f'{project.fp_github_name} up to 5', reviewer)
+    # validate the forward ports for "child", "root", and "parent" so "current"
+    # exists and we have one more target
+    for branch in map(str, range(2, 5+1)):
+        setci(
+            source=source, repo=prod, target=branch,
+            status='failure' if branch == '5' and mode in ('failbump', 'failsucceed') else 'success'
+        )
+        env.run_crons()
+    # cheat: we know PR numbers are assigned sequentially
+    prs = list(map(prod.get_pr, range(1, 6)))
+    before = {p.number: len(p.comments) for p in prs}
+
+    if mode == 'failsucceed':
+        setci(source=source, repo=prod, target=5)
+        # sees the success, limit is still 5, considers the porting finished
+        env.run_crons()
+
+    if mode and mode.startswith('merge'):
+        numbers = range(5 if mode == 'mergetip' else 2, 5 + 1)
+        with prod:
+            for number in numbers:
+                prod.get_pr(number).post_comment(f'{project.github_prefix} r+', reviewer)
+        env.run_crons()
+        with prod:
+            for target in numbers:
+                pr = PRs.search([('target.name', '=', str(target))])
+                print(pr.display_name, pr.state, pr.staging_id)
+                prod.post_status(f'staging.{target}', 'success')
+        env.run_crons()
+        for number in numbers:
+            assert PRs.search([('number', '=', number)]).state == 'merged'
+
+    from_ = prod.get_pr(source.number)
+    with prod:
+        from_.post_comment(f'{project.fp_github_name} up to 6', reviewer)
+    env.run_crons()
+
+    if mode == 'failbump':
+        setci(source=source, repo=prod, target=5)
+        # setci moved the PR from opened to validated, so *now* it can be
+        # forward-ported, but that still needs to actually happen
+        env.run_crons()
+
+    # since PR5 CI succeeded and we've increased the limit there should be a
+    # new PR
+    assert PRs.search([('source_id', '=', source.id), ('target.name', '=', 6)])
+    pr5_id = PRs.search([('source_id', '=', source.id), ('target.name', '=', 5)])
+    if mode == 'failbump':
+        # because the initial forward porting was never finished as the PR CI
+        # failed until *after* we bumped the limit, so it's not *resuming* per se.
+        assert prs[0].comments[before[1]+1:] == [
+            (users['user'], f"Forward-porting to '6'.")
+        ]
+    else:
+        assert prs[0].comments[before[1]+1:] == [
+            (users['user'], f"Forward-porting to '6', resuming forward-port stopped at {pr5_id.display_name}.")
+        ]
+
+def setci(*, source, repo, target, status='success'):
+    """Validates (CI success) the descendant of ``source`` targeting ``target``
+    in  ``repo``.
+    """
+    pr = source.search([('source_id', '=', source.id), ('target.name', '=', str(target))])
+    with repo:
+        repo.post_status(pr.head, status)
+
+
+@pytest.fixture(scope='session')
+def branches():
+    """Need enough branches to make space for:
+
+    - a source
+    - an ancestor (before and separated from the root, but not the source)
+    - a root (break in the parent chain
+    - a parent (between "current" and root)
+    - "current"
+    - the tip branch
+    """
+    return range(1, 6 + 1)
+
+@pytest.fixture
+def post_merge(env, config, users, make_repo, branches):
+    """Create a setup for the post-merge limits test which is both simpler and
+    more complicated than the standard test setup(s): it doesn't need more
+    variety in code, but it needs a lot more "depth" in terms of number of
+    branches it supports. Branches are fixture-ed to make it easier to share
+    between this fixture and the actual test.
+
+    All the branches are set to the same commit because that basically
+    shouldn't matter.
+    """
+    prod = make_repo("post-merge-test")
+    with prod:
+        [c] = prod.make_commits(None, Commit('base', tree={'f': ''}))
+        for i in branches:
+            prod.make_ref(f'heads/{i}', c)
+    dev = prod.fork()
+
+    proj = env['runbot_merge.project'].create({
+        'name': prod.name,
+        'github_token': config['github']['token'],
+        'github_prefix': 'hansen',
+        'fp_github_token': config['github']['token'],
+        'fp_github_name': 'herbert',
+        'fp_github_email': 'hb@example.com',
+        'branch_ids': [
+            (0, 0, {'name': str(i), 'sequence': 1000 - (i * 10)})
+            for i in branches
+        ],
+        'repo_ids': [
+            (0, 0, {
+                'name': prod.name,
+                'required_statuses': 'default',
+                'fp_remote_target': dev.name,
+            })
+        ]
+    })
+
+    env['res.partner'].search([
+        ('github_login', '=', config['role_reviewer']['user'])
+    ]).write({
+        'review_rights': [(0, 0, {'repository_id': proj.repo_ids.id, 'review': True})]
+    })
+
+    mbot = proj.github_prefix
+    reviewer = config['role_reviewer']['token']
+    # merge the source PR
+    source_target = str(branches[0])
+    with prod:
+        [c] = prod.make_commits(source_target, Commit('my pr', tree={'x': ''}), ref='heads/mypr')
+        pr1 = prod.make_pr(target=source_target, head=c, title="a title")
+
+        prod.post_status(c, 'success')
+        pr1.post_comment(f'{mbot} r+', reviewer)
+    env.run_crons()
+    with prod:
+        prod.post_status(f'staging.{source_target}', 'success')
+    env.run_crons()
+
+    return proj, prod, dev
diff --git a/runbot_merge/changelog/2023-10/free-the-limit.md b/runbot_merge/changelog/2023-10/free-the-limit.md
new file mode 100644
index 00000000..4cdee310
--- /dev/null
+++ b/runbot_merge/changelog/2023-10/free-the-limit.md
@@ -0,0 +1,8 @@
+IMP: allow setting forward-port limits after the source pull request has been merged
+
+Should now be possible to both extend and retract the forward port limit
+afterwards, though obviously no shorter than the current tip of the forward
+port sequence. One limitation is that forward ports being created can't be
+stopped so there might be some windows where trying to set the limit to the
+current tip will fail (because it's in the process of being forward-ported to
+the next branch).

From 134ce03053ee917208cfefa145f4cdda9597a15c Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 31 Oct 2023 07:36:10 +0100
Subject: [PATCH 051/192] [FIX] forwardport: fix sourcing on reparenting

When reparenting a commit (mostly when inserting a new forwardport in
an existing chain after a freeze / branch insertion), the new source
should be the source of the new parent (which is likely a not-change
of the source).

This was miscomputed to the root of the new parent, which often
matches but breaks if there was a conflict or a mid-port update,
leading to inconsistent presentation. Nothing critical, just somewhat
annoying.
---
 forwardport/models/project.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index adde4192..ff977628 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -348,7 +348,8 @@ class PullRequests(models.Model):
                 })
 
         if vals.get('parent_id') and 'source_id' not in vals:
-            vals['source_id'] = self.browse(vals['parent_id']).root_id.id
+            parent = self.browse(vals['parent_id'])
+            vals['source_id'] = (parent.source_id or parent).id
         if vals.get('state') == 'merged':
             vals['merge_date'] = fields.Datetime.now()
         r = super().write(vals)

From a15086a8a91570176e111a1c16e81b3c625f551a Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 27 Oct 2023 11:15:05 +0200
Subject: [PATCH 052/192] [FIX] runbot_merge: "not something we can merge"
 freeze error

During the 17.0 freezeathon, the freeze wizard blew up with

    MergeError: merge-tree: {oid} - not something we can merge

Turns out when freezes were moved to local
(4d2c0f86e1f3e058f81aabad832a666739d980e3) I forgot to fetch the heads
of the release and bump PRs into the local repo, so rebasing them atop
their branch would fail because the local repository would just not
find the object being rebased.

I had missed that case in testing as well, but in fairness even if I
had tried testing it I'd likely have missed it: implementation
limitations (shortcuts) of dummy central mean it currently ignores
what objects the client requests and bundles everything it can find
associated with the repository (meaning it sends the entire network).

This is not usually an issue because the test repos are pretty small,
but it means the client can have objects they should not because they
never requested them and might not even be supposed to be aware of
their existence.

Anyway solve by doing the obvious: fetch the heads of the release and
bump PRs at the same time we update the branch being forked off. Also
update the freeze tests to trigger the issue (by creating the release
/ bump PRs in different repos) and running the tests against github
actual to make sure we can actually see them fail (correctly, the
merge error we expect) not via errors in the test), and we do fix
them.

Fixes #821
---
 .../models/project_freeze/__init__.py         |  5 ++
 runbot_merge/tests/test_multirepo.py          | 57 ++++++++++---------
 2 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py
index 7684249d..77cffc96 100644
--- a/runbot_merge/models/project_freeze/__init__.py
+++ b/runbot_merge/models/project_freeze/__init__.py
@@ -218,6 +218,11 @@ class FreezeWizard(models.Model):
         }
         for repo, copy in repos.items():
             copy.fetch(git.source_url(repo, 'github'), '+refs/heads/*:refs/heads/*')
+        for pr in self.release_pr_ids.pr_id | self.bump_pr_ids.pr_id:
+            repos[pr.repository].fetch(
+                git.source_url(pr.repository, 'github'),
+                pr.head,
+            )
 
         # prep new branch (via tmp refs) on every repo
         rel_heads: Dict[Repository, str] = {}
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 59b3f932..3ba105ca 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -1171,20 +1171,19 @@ def test_freeze_complete(env, project, repo_a, repo_b, repo_c, users, config):
     ]})
     r = w_id.action_freeze()
     assert r == w, "the freeze is not ready so the wizard should redirect to itself"
-    owner = repo_c.owner
     assert w_id.errors == f"""\
-* All release PRs must have the same label, found '{owner}:release-1.1, {owner}:whocares'.
+* All release PRs must have the same label, found '{pr_rel_c.user}:release-1.1, {pr_other.user}:whocares'.
 * 2 required PRs not ready."""
     w_id.release_pr_ids[-1].pr_id = release_prs[repo_c.name].id
 
     with repo_a:
         pr_required_a.post_comment('hansen r+', config['role_reviewer']['token'])
-        repo_a.post_status('apr', 'success', 'ci/runbot')
-        repo_a.post_status('apr', 'success', 'legal/cla')
+        repo_a.post_status(pr_required_a.head, 'success', 'ci/runbot')
+        repo_a.post_status(pr_required_a.head, 'success', 'legal/cla')
     with repo_c:
         pr_required_c.post_comment('hansen r+', config['role_reviewer']['token'])
-        repo_c.post_status('cpr', 'success', 'ci/runbot')
-        repo_c.post_status('cpr', 'success', 'legal/cla')
+        repo_c.post_status(pr_required_c.head, 'success', 'ci/runbot')
+        repo_c.post_status(pr_required_c.head, 'success', 'legal/cla')
     env.run_crons()
 
     for repo in [repo_a, repo_b, repo_c]:
@@ -1275,45 +1274,51 @@ def setup_mess(repo_a, repo_b, repo_c):
                 ref='heads/1.0'
             )
             master_heads.extend(r.make_commits(root, Commit('other', tree={'f': '1'}), ref='heads/master'))
+
+    a_fork = repo_a.fork()
+    b_fork = repo_b.fork()
+    c_fork = repo_c.fork()
+    assert a_fork.owner == b_fork.owner == c_fork.owner
+    owner = a_fork.owner
     # have 2 PRs required for the freeze
-    with repo_a:
-        repo_a.make_commits(master_heads[0], Commit('super important file', tree={'g': 'x'}), ref='heads/apr')
-        pr_required_a = repo_a.make_pr(target='master', head='apr')
-    with repo_c:
-        repo_c.make_commits(master_heads[2], Commit('update thing', tree={'f': '2'}), ref='heads/cpr')
-        pr_required_c = repo_c.make_pr(target='master', head='cpr')
+    with repo_a, a_fork:
+        a_fork.make_commits(master_heads[0], Commit('super important file', tree={'g': 'x'}), ref='heads/apr')
+        pr_required_a = repo_a.make_pr(target='master', head=f'{owner}:apr', title="xxx")
+    with repo_c, c_fork:
+        c_fork.make_commits(master_heads[2], Commit('update thing', tree={'f': '2'}), ref='heads/cpr')
+        pr_required_c = repo_c.make_pr(target='master', head=f'{owner}:cpr', title="yyy")
     # have 3 release PRs, only the first one updates the tree (version file)
-    with repo_a:
-        repo_a.make_commits(
+    with repo_a, a_fork:
+        a_fork.make_commits(
             master_heads[0],
             Commit('Release 1.1 (A)', tree={'version': '1.1'}),
             ref='heads/release-1.1'
         )
-        pr_rel_a = repo_a.make_pr(target='master', head='release-1.1')
-    with repo_b:
-        repo_b.make_commits(
+        pr_rel_a = repo_a.make_pr(target='master', head=f'{owner}:release-1.1', title="zzz")
+    with repo_b, b_fork:
+        b_fork.make_commits(
             master_heads[1],
             Commit('Release 1.1 (B)', tree={'version': '1.1'}),
             ref='heads/release-1.1'
         )
-        pr_rel_b = repo_b.make_pr(target='master', head='release-1.1')
-    with repo_c:
-        repo_c.make_commits(master_heads[2], Commit("Some change", tree={'a': '1'}), ref='heads/whocares')
-        pr_other = repo_c.make_pr(target='master', head='whocares')
-        repo_c.make_commits(
+        pr_rel_b = repo_b.make_pr(target='master', head=f'{owner}:release-1.1', title="000")
+    with repo_c, c_fork:
+        c_fork.make_commits(master_heads[2], Commit("Some change", tree={'a': '1'}), ref='heads/whocares')
+        pr_other = repo_c.make_pr(target='master', head=f'{owner}:whocares', title="111")
+        c_fork.make_commits(
             master_heads[2],
             Commit('Release 1.1 (C)', tree={'version': '1.1'}),
             ref='heads/release-1.1'
         )
-        pr_rel_c = repo_c.make_pr(target='master', head='release-1.1')
+        pr_rel_c = repo_c.make_pr(target='master', head=f'{owner}:release-1.1', title="222")
     # have one bump PR on repo A
-    with repo_a:
-        repo_a.make_commits(
+    with repo_a, a_fork:
+        a_fork.make_commits(
             master_heads[0],
             Commit("Bump A", tree={'version': '1.2-alpha'}),
             ref='heads/bump-1.1',
         )
-        pr_bump_a = repo_a.make_pr(target='master', head='bump-1.1')
+        pr_bump_a = repo_a.make_pr(target='master', head=f'{owner}:bump-1.1', title="333")
     return master_heads, (pr_required_a, None, pr_required_c), (pr_rel_a, pr_rel_b, pr_rel_c), pr_bump_a, pr_other
 
 def test_freeze_subset(env, project, repo_a, repo_b, repo_c, users, config):

From 2cd3fb8999bdc349cf2d5b1f0786d34196588b1e Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 6 Oct 2023 15:19:36 +0200
Subject: [PATCH 053/192] [IMP] runbot_merge: make uniquifier commit optional

Prepares the possibility of either more direct communication with the
CI platform(s) or just assuming CI has gotten reliable enough and
colleagues intelligent enough that this is not an issue anymore
because they've stopped pushing empty branches (which we know is not
the case).

Fixes #806
---
 runbot_merge/models/project.py              |  9 ++++++
 runbot_merge/models/stagings_create.py      | 34 ++++++++++-----------
 runbot_merge/tests/test_multirepo.py        |  9 ++++--
 runbot_merge/views/runbot_merge_project.xml |  1 +
 4 files changed, 34 insertions(+), 19 deletions(-)

diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index e828d29b..57fc1a33 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -51,6 +51,15 @@ class Project(models.Model):
     freeze_id = fields.Many2one('runbot_merge.project.freeze', compute='_compute_freeze')
     freeze_reminder = fields.Text()
 
+    uniquifier = fields.Boolean(
+        default=True,
+        help="Whether to add a uniquifier commit on repositories without PRs"
+             " during staging. The lack of uniquifier can lead to CI conflicts"
+             " as github works off of commits, so it's possible for an"
+             " unrelated build to trigger a failure if somebody is a dummy and"
+             " includes repos they have no commit for."
+    )
+
     @api.depends('github_token')
     def _compute_identity(self):
         s = requests.Session()
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index 42b1aad5..7e083e0e 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -90,23 +90,11 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
     heads = []
     commits = []
     for repo, it in staging_state.items():
-        if it.head != original_heads[repo]:
-            # if we staged something for that repo, just create a record for
-            # that commit, or flag existing one as to-recheck in case there are
-            # already statuses we want to propagate to the staging or something
-            env.cr.execute(
-                "INSERT INTO runbot_merge_commit (sha, to_check, statuses) "
-                "VALUES (%s, true, '{}') "
-                "ON CONFLICT (sha) DO UPDATE SET to_check=true "
-                "RETURNING id",
-                [it.head]
-            )
-            [commit] = [head] = env.cr.fetchone()
-        else:
-            # if we didn't stage anything for that repo, create a dummy commit
-            # (with a uniquifier to ensure we don't hit a previous version of
-            # the same) to ensure the staging head is new and we're building
-            # everything
+        if it.head == original_heads[repo] and branch.project_id.uniquifier:
+            # if we didn't stage anything for that repo and uniquification is
+            # enabled, create a dummy commit with a uniquifier to ensure we
+            # don't hit a previous version of the same to ensure the staging
+            # head is new and we're building everything
             project = branch.project_id
             uniquifier = base64.b64encode(os.urandom(12)).decode('ascii')
             dummy_head = it.repo.with_config(check=True).commit_tree(
@@ -135,6 +123,18 @@ For-Commit-Id: {it.head}
             )
             ([commit], [head]) = env.cr.fetchall()
             it.head = dummy_head
+        else:
+            # otherwise just create a record for that commit, or flag existing
+            # one as to-recheck in case there are already statuses we want to
+            # propagate to the staging or something
+            env.cr.execute(
+                "INSERT INTO runbot_merge_commit (sha, to_check, statuses) "
+                "VALUES (%s, true, '{}') "
+                "ON CONFLICT (sha) DO UPDATE SET to_check=true "
+                "RETURNING id",
+                [it.head]
+            )
+            [commit] = [head] = env.cr.fetchone()
 
         heads.append(fields.Command.create({
             'repository_id': repo.id,
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 3ba105ca..dbae165c 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -87,9 +87,11 @@ def make_branch(repo, name, message, tree, protect=True):
         repo.protect(name)
     return c
 
-def test_stage_one(env, project, repo_a, repo_b, config):
+@pytest.mark.parametrize('uniquifier', [False, True])
+def test_stage_one(env, project, repo_a, repo_b, config, uniquifier):
     """ First PR is non-matched from A => should not select PR from B
     """
+    project.uniquifier = uniquifier
     project.batch_limit = 1
 
     with repo_a:
@@ -112,7 +114,10 @@ def test_stage_one(env, project, repo_a, repo_b, config):
     assert pra_id.state == 'ready'
     assert pra_id.staging_id
     assert repo_a.commit('staging.master').message.startswith('commit_A_00')
-    assert repo_b.commit('staging.master').message.startswith('force rebuild')
+    if uniquifier:
+        assert repo_b.commit('staging.master').message.startswith('force rebuild')
+    else:
+        assert repo_b.commit('staging.master').message == 'initial'
 
     prb_id = to_pr(env, pr_b)
     assert prb_id.state == 'ready'
diff --git a/runbot_merge/views/runbot_merge_project.xml b/runbot_merge/views/runbot_merge_project.xml
index 4795b502..5aa49d47 100644
--- a/runbot_merge/views/runbot_merge_project.xml
+++ b/runbot_merge/views/runbot_merge_project.xml
@@ -32,6 +32,7 @@
                             <field name="secret"/>
                         </group>
                         <group>
+                            <field name="uniquifier"/>
                             <field name="ci_timeout"/>
                             <field name="batch_limit"/>
                         </group>

From b21fbaf9cc18b4a1cbc18e12ca9703091a8ef521 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 9 Oct 2023 13:52:30 +0200
Subject: [PATCH 054/192] [IMP] runbot_merge: prevent merging empty commits

The low-level APIs used by the staging process don't do any merge
check, so because of the way Git works it's possible for them to merge
commits with content as empty commits, e.g. if something was merged
then backported and the backport was merged on top. This should
trigger a merge failure as we don't really want to merge newly
empty. This is a feature which some high level commands of git
support, kind-of, e.g. by default `git rebase --interactive` will ask
about newly empty commits.

Take care to allow merging already-empty commits, as these do have a
use for signaling, freezes, ....

Fixes #809
---
 runbot_merge/git.py                    |  19 +++++
 runbot_merge/github.py                 |   6 +-
 runbot_merge/models/stagings_create.py |  13 ++-
 runbot_merge/tests/test_oddities.py    | 109 +++++++++++++++++++++++++
 4 files changed, 145 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/git.py b/runbot_merge/git.py
index 09061933..f8bdb7f5 100644
--- a/runbot_merge/git.py
+++ b/runbot_merge/git.py
@@ -107,6 +107,11 @@ class Repo:
         )
         return Repo(to)
 
+    def get_tree(self, commit_hash: str) -> str:
+        r = self.with_config(check=True).rev_parse(f'{commit_hash}^{{tree}}')
+
+        return r.stdout.strip()
+
     def rebase(self, dest: str, commits: Sequence[PrCommit]) -> Tuple[str, Dict[str, str]]:
         """Implements rebase by hand atop plumbing so:
 
@@ -125,6 +130,9 @@ class Repo:
         if not commits:
             raise MergeError("PR has no commits")
 
+        prev_tree = repo.get_tree(dest)
+        prev_original_tree = repo.get_tree(commits[0]['parents'][0]["sha"])
+
         new_trees = []
         parent = dest
         for original in commits:
@@ -135,11 +143,22 @@ class Repo:
                     "rebasing")
 
             new_trees.append(check(repo.merge_tree(parent, original['sha'])).stdout.strip())
+            # allow merging empty commits, but not empty*ing* commits while merging
+            if prev_original_tree != original['commit']['tree']['sha']:
+                if new_trees[-1] == prev_tree:
+                    raise MergeError(
+                        f"commit {original['sha']} results in an empty tree when "
+                        f"merged, it is likely a duplicate of a merged commit, "
+                        f"rebase and remove."
+                    )
+
             parent = check(repo.commit_tree(
                 tree=new_trees[-1],
                 parents=[parent, original['sha']],
                 message=f'temp rebase {original["sha"]}',
             )).stdout.strip()
+            prev_tree = new_trees[-1]
+            prev_original_tree = original['commit']['tree']['sha']
 
         mapping = {}
         for original, tree in zip(commits, new_trees):
diff --git a/runbot_merge/github.py b/runbot_merge/github.py
index 1f31e03c..2abcc495 100644
--- a/runbot_merge/github.py
+++ b/runbot_merge/github.py
@@ -57,8 +57,12 @@ Authorship = TypedDict('Authorship', {
     'name': str,
     'email': str,
 })
+CommitTree = TypedDict('CommitTree', {
+    'sha': str,
+    'url': str,
+})
 Commit = TypedDict('Commit', {
-    'tree': str,
+    'tree': CommitTree,
     'url': str,
     'message': str,
     # optional when creating a commit
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index 7e083e0e..382fa49d 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -426,7 +426,18 @@ def stage(pr: PullRequests, info: StagingSlice, related_prs: PullRequests) -> Tu
             fn = stage_rebase_ff
         case 'squash':
             fn = stage_squash
-    return method, fn(pr, info, pr_commits, related_prs=related_prs)
+
+    pr_base_tree = info.repo.get_tree(pr_commits[0]['parents'][0]['sha'])
+    pr_head_tree = pr_commits[-1]['commit']['tree']['sha']
+
+    merge_base_tree = info.repo.get_tree(info.head)
+    new_head = fn(pr, info, pr_commits, related_prs=related_prs)
+    merge_head_tree = info.repo.get_tree(new_head)
+
+    if pr_head_tree != pr_base_tree and merge_head_tree == merge_base_tree:
+        raise exceptions.MergeError(pr, f'results in an empty tree when merged, might be the duplicate of a merged PR.')
+
+    return method, new_head
 
 def stage_squash(pr: PullRequests, info: StagingSlice, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
     msg = pr._build_merge_message(pr, related_prs=related_prs)
diff --git a/runbot_merge/tests/test_oddities.py b/runbot_merge/tests/test_oddities.py
index d8121629..755a3312 100644
--- a/runbot_merge/tests/test_oddities.py
+++ b/runbot_merge/tests/test_oddities.py
@@ -143,3 +143,112 @@ def test_staging_post_update(env, project, make_repo, setreviewers, users, confi
     assert staging_id.statuses == [
         [repo.name, 'ci/runbot', 'failure', ''],
     ]
+
+def test_merge_empty_commits(env, project, make_repo, setreviewers, users, config):
+    """The mergebot should allow merging already-empty commits.
+    """
+    repo = make_repo('repo')
+    project.write({'repo_ids': [(0, 0, {
+        'name': repo.name,
+        'group_id': False,
+        'required_statuses': 'default',
+    })]})
+    setreviewers(*project.repo_ids)
+
+    with repo:
+        [m] = repo.make_commits(None, Commit('initial', tree={'m': 'm'}), ref='heads/master')
+
+        repo.make_commits(m, Commit('thing1', tree={}), ref='heads/other1')
+        pr1 = repo.make_pr(target='master', head='other1')
+        repo.post_status(pr1.head, 'success')
+        pr1.post_comment('hansen r+', config['role_reviewer']['token'])
+
+        repo.make_commits(m, Commit('thing2', tree={}), ref='heads/other2')
+        pr2 = repo.make_pr(target='master', head='other2')
+        repo.post_status(pr2.head, 'success')
+        pr2.post_comment('hansen r+ rebase-ff', config['role_reviewer']['token'])
+    env.run_crons()
+    pr1_id = to_pr(env, pr1)
+    pr2_id = to_pr(env, pr2)
+    assert pr1_id.staging_id and pr2_id.staging_id
+
+    with repo:
+        repo.post_status('staging.master', 'success')
+    env.run_crons()
+
+    assert pr1_id.state == pr2_id.state == 'merged'
+
+    # log is most-recent-first (?)
+    commits = list(repo.log('master'))
+    head = repo.commit(commits[0]['sha'])
+    assert repo.read_tree(head) == {'m': 'm'}
+
+    assert commits[0]['commit']['message'].startswith('thing2')
+    assert commits[1]['commit']['message'].startswith('thing1')
+    assert commits[2]['commit']['message'] == 'initial'
+
+def test_merge_emptying_commits(env, project, make_repo, setreviewers, users, config):
+    """The mergebot should *not* allow merging non-empty commits which become
+    empty as part of the staging (rebasing)
+    """
+    repo = make_repo('repo')
+    project.write({'repo_ids': [(0, 0, {
+        'name': repo.name,
+        'group_id': False,
+        'required_statuses': 'default',
+    })]})
+    setreviewers(*project.repo_ids)
+
+    with repo:
+        [m, _] = repo.make_commits(
+            None,
+            Commit('initial', tree={'m': 'm'}),
+            Commit('second', tree={'m': 'c'}),
+            ref='heads/master',
+        )
+
+        [c1] = repo.make_commits(m, Commit('thing', tree={'m': 'c'}), ref='heads/branch1')
+        pr1 = repo.make_pr(target='master', head='branch1')
+        repo.post_status(pr1.head, 'success')
+        pr1.post_comment('hansen r+ rebase-ff', config['role_reviewer']['token'])
+
+        [_, c2] = repo.make_commits(
+            m,
+            Commit('thing1', tree={'c': 'c'}),
+            Commit('thing2', tree={'m': 'c'}),
+            ref='heads/branch2',
+        )
+        pr2 = repo.make_pr(target='master', head='branch2')
+        repo.post_status(pr2.head, 'success')
+        pr2.post_comment('hansen r+ rebase-ff', config['role_reviewer']['token'])
+
+        repo.make_commits(
+            m,
+            Commit('thing1', tree={'m': 'x'}),
+            Commit('thing2', tree={'m': 'c'}),
+            ref='heads/branch3',
+        )
+        pr3 = repo.make_pr(target='master', head='branch3')
+        repo.post_status(pr3.head, 'success')
+        pr3.post_comment('hansen r+ squash', config['role_reviewer']['token'])
+    env.run_crons()
+
+    ping = f"@{users['user']} @{users['reviewer']}"
+    # check that first / sole commit emptying is caught
+    assert not to_pr(env, pr1).staging_id
+    assert pr1.comments[3:] == [
+        (users['user'], f"{ping} unable to stage: commit {c1} results in an empty tree when merged, it is likely a duplicate of a merged commit, rebase and remove.")
+    ]
+
+    # check that followup commit emptying is caught
+    assert not to_pr(env, pr2).staging_id
+    assert pr2.comments[3:] == [
+        (users['user'], f"{ping} unable to stage: commit {c2} results in an empty tree when merged, it is likely a duplicate of a merged commit, rebase and remove.")
+    ]
+
+    # check that emptied squashed pr is caught
+    pr3_id = to_pr(env, pr3)
+    assert not pr3_id.staging_id
+    assert pr3.comments[3:] == [
+        (users['user'], f"{ping} unable to stage: results in an empty tree when merged, might be the duplicate of a merged PR.")
+    ]

From 994cea467c2e8f0576b11fed4629e2b7f608db82 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 16 Jan 2024 07:54:43 +0100
Subject: [PATCH 055/192] [FIX] runbot_merge: typo in freeze wizard

Forgot to deref the id of the staging we're trying to lock, so the
specific case where we start a freeze with a bump PR and an
outstanding staging in master would instantly blow up.
---
 runbot_merge/models/project_freeze/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py
index 77cffc96..2a0961bf 100644
--- a/runbot_merge/models/project_freeze/__init__.py
+++ b/runbot_merge/models/project_freeze/__init__.py
@@ -196,7 +196,7 @@ class FreezeWizard(models.Model):
         if self.bump_pr_ids and master.active_staging_id:
             self.env.cr.execute(
                 'SELECT * FROM runbot_merge_stagings WHERE id = %s FOR UPDATE NOWAIT',
-                [master.active_staging_id]
+                [master.active_staging_id.id]
             )
 
         seq = itertools.count(start=1) # start reseq at 1

From 1cb31cf2c2f4a914bc946a491c7a2b3d132929d9 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 16 Jan 2024 09:42:57 +0100
Subject: [PATCH 056/192] [FIX] runbot_merge: 1.9 version & migration

Forgot to bump the version when creating the migration. Also convert
the migration to a single sql query, although the migration will never
run because I ran the query manually to fix things up after finding
out the data was "dirty" since the new code (assuming only modern
statuses) was merged without running the migration.

Thankfully it looks like the impact was not too severe (because the
legacy statuses should only be present on very old commits / PRs), I
don't remember when I deployed the update but apparently just a pair
of PRs got affected, because their `previous_failure` was the old
style and thus broke the "new failure" check.
---
 runbot_merge/__manifest__.py                  |  2 +-
 .../migrations/15.0.1.9/pre-migration.py      | 36 +++++++++----------
 2 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py
index 266605d3..d36aea6e 100644
--- a/runbot_merge/__manifest__.py
+++ b/runbot_merge/__manifest__.py
@@ -1,6 +1,6 @@
 {
     'name': 'merge bot',
-    'version': '1.8',
+    'version': '1.9',
     'depends': ['contacts', 'website'],
     'data': [
         'security/security.xml',
diff --git a/runbot_merge/migrations/15.0.1.9/pre-migration.py b/runbot_merge/migrations/15.0.1.9/pre-migration.py
index c8c31582..76539967 100644
--- a/runbot_merge/migrations/15.0.1.9/pre-migration.py
+++ b/runbot_merge/migrations/15.0.1.9/pre-migration.py
@@ -9,28 +9,24 @@ def migrate(cr, version):
     # style is like commit statuses, with the contexts at the toplevel
     # and the status info below.
     cr.execute("""
-UPDATE runbot_merge
+UPDATE runbot_merge_pull_requests
    SET previous_failure = '{}'
  WHERE previous_failure::jsonb ? 'state'
 """)
 
-    # Getting this into postgres native manipulations seems a bit too
-    # complicated, and not really necessary.
     cr.execute("""
-SELECT id, statuses::json
-  FROM runbot_merge_commit
- WHERE jsonb_path_match(statuses::jsonb, '$.*.type() != "object"')
-""")
-    updated = [
-        (id, {
-            k: {'state': r, 'target_url': None, 'description': None}
-            for k, r in st.items()
-        })
-        for id, st in cr.fetchall()
-    ]
-    execute_values(cr._obj, """
-UPDATE runbot_merge_commit c
-   SET c.statuses = data.st
-  FROM (VALUES %s) AS data (id, st)
- WHERE c.id = data.id
-""", updated)
+WITH new_statuses (id, statuses) AS (
+    SELECT id, json_object_agg(
+        key,
+        CASE WHEN jsonb_typeof(value) = 'string'
+            THEN jsonb_build_object('state', value, 'target_url', null, 'description', null)
+        ELSE value
+        END
+    ) AS statuses
+    FROM runbot_merge_commit
+    CROSS JOIN LATERAL jsonb_each(statuses::jsonb) s
+    WHERE jsonb_path_match(statuses::jsonb, '$.*.type() != "object"')
+    GROUP BY id
+)
+UPDATE runbot_merge_commit SET statuses = new_statuses.statuses FROM new_statuses WHERE runbot_merge_commit.id = new_statuses.id
+    """)

From 45f0c8cc8192960dae45fc6dd167cb5c9c31b5f9 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 16 Jan 2024 09:45:49 +0100
Subject: [PATCH 057/192] [FIX] runbot_merge: rebase logging

The logging line was copied over from the github-api version, but it
was not correctly fixed up to match, leading to a lot of spam on
stderr when debug is enabled (aka spams journalctl on the production
server).

Splat the logging call out of `rebase` and into the various callers,
so they have access to the pr object to log it.
---
 runbot_merge/git.py                            | 2 --
 runbot_merge/models/project_freeze/__init__.py | 4 ++++
 runbot_merge/models/stagings_create.py         | 5 +++++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/git.py b/runbot_merge/git.py
index f8bdb7f5..ead0e6f1 100644
--- a/runbot_merge/git.py
+++ b/runbot_merge/git.py
@@ -125,8 +125,6 @@ class Repo:
         repo = self.stdout().with_config(text=True, check=False)
 
         logger = _logger.getChild('rebase')
-        logger.debug("rebasing %s on %s (reset=%s, commits=%s)",
-                     self._repo, dest, len(commits))
         if not commits:
             raise MergeError("PR has no commits")
 
diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py
index 2a0961bf..c5b3312e 100644
--- a/runbot_merge/models/project_freeze/__init__.py
+++ b/runbot_merge/models/project_freeze/__init__.py
@@ -242,6 +242,8 @@ class FreezeWizard(models.Model):
             except Exception as e:
                 raise UserError(f"Unable to fetch commits of release PR {rel.pr_id.display_name}.") from e
 
+            _logger.debug("rebasing %s on %s (commits=%s)",
+                          rel.pr_id.display_name, prev, len(commits))
             rel_heads[repo_id] = repos[repo_id].rebase(prev, commits)[0]
 
         # prep bump
@@ -260,6 +262,8 @@ class FreezeWizard(models.Model):
             except Exception as e:
                 raise UserError(f"Unable to fetch commits of bump PR {bump.pr_id.display_name}.") from e
 
+            _logger.debug("rebasing %s on %s (commits=%s)",
+                          bump.pr_id.display_name, prev, len(commits))
             bump_heads[repo_id] = repos[repo_id].rebase(prev, commits)[0]
 
         deployed = {}
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index 382fa49d..f426db7d 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -490,12 +490,17 @@ def stage_rebase_ff(pr: PullRequests, info: StagingSlice, commits: List[github.P
     msg = pr._build_merge_message(commits[-1]['commit']['message'], related_prs=related_prs)
     commits[-1]['commit']['message'] = str(msg)
     add_self_references(pr, commits[:-1])
+
+    _logger.debug("rebasing %s on %s (commits=%s)",
+                  pr.display_name, info.head, len(commits))
     head, mapping = info.repo.rebase(info.head, commits=commits)
     pr.commits_map = json.dumps({**mapping, '': head})
     return head
 
 def stage_rebase_merge(pr: PullRequests, info: StagingSlice, commits: List[github.PrCommit], related_prs: PullRequests) -> str :
     add_self_references(pr, commits)
+    _logger.debug("rebasing %s on %s (commits=%s)",
+                  pr.display_name, info.head, len(commits))
     h, mapping = info.repo.rebase(info.head, commits=commits)
     msg = pr._build_merge_message(pr, related_prs=related_prs)
 

From cea1b62ac22343ce62858e4810637b89eb4f71fd Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 16 Jan 2024 10:51:37 +0100
Subject: [PATCH 058/192] [FIX] runbot_merge: commit messages should be trimmed
 indeed

Reverts commit 85a78900239d793e3fd62dcfbf7914f9e223a5ea which
untrimmed the commits: while it's *probably* true that git and
github's APIs differ in their treatment of whitespace (in that git
pretty much always terminates the commit message with a newline while
github does not, as far as I understand, though I didn't really
validate it) the issue was that github also trims on *output* when
fetching over the API, something the fake did not do.

So rather than update the test data I should have fixed the fake, but
I failed to realise that at the time. I only realised when I decided
to re-run against github actual (something I rarely do anymore as it's
painfully slow) and it went on to choke on every message I'd updated.
---
 mergebot_test_utils/utils.py         |   7 +-
 runbot_merge/tests/test_basic.py     | 104 +++++++++++++--------------
 runbot_merge/tests/test_multirepo.py |  12 ++--
 3 files changed, 60 insertions(+), 63 deletions(-)

diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index 922eb437..5f61bbb1 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -8,8 +8,7 @@ MESSAGE_TEMPLATE = """{message}
 
 closes {repo}#{number}
 
-{headers}Signed-off-by: {name} <{email}>
-"""
+{headers}Signed-off-by: {name} <{email}>"""
 # target branch '-' source branch '-' base64 unique '-fw'
 REF_PATTERN = r'{target}-{source}-[a-zA-Z0-9_-]{{4}}-fw'
 
@@ -137,4 +136,6 @@ def to_pr(env, pr):
     return pr
 
 def part_of(label, pr_id, *, separator='\n\n'):
-    return f'{label}{separator}Part-of: {pr_id.display_name}\n'
+    """ Adds the "part-of" pseudo-header in the footer.
+    """
+    return f'{label}{separator}Part-of: {pr_id.display_name}'
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index dba0b4b8..1194283e 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -1,6 +1,7 @@
 import datetime
 import itertools
 import json
+import textwrap
 import time
 from unittest import mock
 
@@ -117,7 +118,7 @@ def test_trivial_flow(env, repo, page, users, config):
         'b': 'a second file',
     }
     assert master.message == "gibberish\n\nblahblah\n\ncloses {repo.name}#1"\
-                             "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
+                             "\n\nSigned-off-by: {reviewer.formatted_email}"\
                              .format(repo=repo, reviewer=get_partner(env, users['reviewer']))
 
 class TestCommitMessage:
@@ -142,7 +143,7 @@ class TestCommitMessage:
 
         master = repo.commit('heads/master')
         assert master.message == "simple commit message\n\ncloses {repo.name}#1"\
-                                 "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
+                                 "\n\nSigned-off-by: {reviewer.formatted_email}"\
                                  .format(repo=repo, reviewer=get_partner(env, users['reviewer']))
 
     def test_commit_existing(self, env, repo, users, config):
@@ -167,7 +168,7 @@ class TestCommitMessage:
         master = repo.commit('heads/master')
         # closes #1 is already present, should not modify message
         assert master.message == "simple commit message that closes #1"\
-                                 "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
+                                 "\n\nSigned-off-by: {reviewer.formatted_email}"\
                                  .format(reviewer=get_partner(env, users['reviewer']))
 
     def test_commit_other(self, env, repo, users, config):
@@ -192,7 +193,7 @@ class TestCommitMessage:
         master = repo.commit('heads/master')
         # closes on another repositoy, should modify the commit message
         assert master.message == "simple commit message that closes odoo/enterprise#1\n\ncloses {repo.name}#1"\
-                                 "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
+                                 "\n\nSigned-off-by: {reviewer.formatted_email}"\
                                  .format(repo=repo, reviewer=get_partner(env, users['reviewer']))
 
     def test_commit_wrong_number(self, env, repo, users, config):
@@ -217,7 +218,7 @@ class TestCommitMessage:
         master = repo.commit('heads/master')
         # closes on another repositoy, should modify the commit message
         assert master.message == "simple commit message that closes #11\n\ncloses {repo.name}#1"\
-                                 "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
+                                 "\n\nSigned-off-by: {reviewer.formatted_email}"\
                                  .format(repo=repo, reviewer=get_partner(env, users['reviewer']))
 
     def test_commit_delegate(self, env, repo, users, config):
@@ -247,7 +248,7 @@ class TestCommitMessage:
 
         master = repo.commit('heads/master')
         assert master.message == "simple commit message\n\ncloses {repo.name}#1"\
-                                 "\n\nSigned-off-by: {reviewer.formatted_email}\n"\
+                                 "\n\nSigned-off-by: {reviewer.formatted_email}"\
                                  .format(repo=repo, reviewer=get_partner(env, users['other']))
 
     def test_commit_coauthored(self, env, repo, users, config):
@@ -285,8 +286,7 @@ Fixes a thing
 closes {repo.name}#1
 
 Signed-off-by: {reviewer.formatted_email}
-Co-authored-by: Bob <bob@example.com>
-""".format(
+Co-authored-by: Bob <bob@example.com>""".format(
             repo=repo,
             reviewer=get_partner(env, users['reviewer'])
         )
@@ -695,9 +695,9 @@ def test_ff_failure_batch(env, repo, users, config):
     reviewer = get_partner(env, users["reviewer"]).formatted_email
     assert messages == {
         'initial', 'NO!',
-        part_of('a1', pr_a), part_of('a2', pr_a), f'A\n\ncloses {pr_a.display_name}\n\nSigned-off-by: {reviewer}\n',
-        part_of('b1', pr_b), part_of('b2', pr_b), f'B\n\ncloses {pr_b.display_name}\n\nSigned-off-by: {reviewer}\n',
-        part_of('c1', pr_c), part_of('c2', pr_c), f'C\n\ncloses {pr_c.display_name}\n\nSigned-off-by: {reviewer}\n',
+        part_of('a1', pr_a), part_of('a2', pr_a), f'A\n\ncloses {pr_a.display_name}\n\nSigned-off-by: {reviewer}',
+        part_of('b1', pr_b), part_of('b2', pr_b), f'B\n\ncloses {pr_b.display_name}\n\nSigned-off-by: {reviewer}',
+        part_of('c1', pr_c), part_of('c2', pr_c), f'C\n\ncloses {pr_c.display_name}\n\nSigned-off-by: {reviewer}',
     }
 
 class TestPREdition:
@@ -1526,7 +1526,7 @@ commits, I need to know how to merge it:
         nb1 = node(part_of('B1', pr_id), node(part_of('B0', pr_id), nm2))
         reviewer = get_partner(env, users["reviewer"]).formatted_email
         merge_head = (
-            f'title\n\nbody\n\ncloses {pr_id.display_name}\n\nSigned-off-by: {reviewer}\n',
+            f'title\n\nbody\n\ncloses {pr_id.display_name}\n\nSigned-off-by: {reviewer}',
             frozenset([nm2, nb1])
         )
         assert staging == merge_head
@@ -1622,7 +1622,7 @@ commits, I need to know how to merge it:
         # then compare to the dag version of the right graph
         nm2 = node('M2', node('M1', node('M0')))
         reviewer = get_partner(env, users["reviewer"]).formatted_email
-        nb1 = node(f'B1\n\ncloses {pr_id.display_name}\n\nSigned-off-by: {reviewer}\n',
+        nb1 = node(f'B1\n\ncloses {pr_id.display_name}\n\nSigned-off-by: {reviewer}',
                    node(part_of('B0', pr_id), nm2))
         assert staging == nb1
 
@@ -1712,7 +1712,7 @@ commits, I need to know how to merge it:
         c0 = node('C0', m)
         reviewer = get_partner(env, users["reviewer"]).formatted_email
         expected = node('gibberish\n\nblahblah\n\ncloses {}#{}'
-                        '\n\nSigned-off-by: {}\n'.format(repo.name, prx.number, reviewer), m, c0)
+                        '\n\nSigned-off-by: {}'.format(repo.name, prx.number, reviewer), m, c0)
         assert log_to_node(repo.log('heads/master')), expected
         pr = env['runbot_merge.pull_requests'].search([
             ('repository.name', '=', repo.name),
@@ -1754,7 +1754,7 @@ commits, I need to know how to merge it:
         c0 = node('C0', m)
         reviewer = get_partner(env, users["reviewer"]).formatted_email
         expected = node('gibberish\n\ncloses {}#{}'
-                        '\n\nSigned-off-by: {}\n'.format(repo.name, prx.number, reviewer), m, c0)
+                        '\n\nSigned-off-by: {}'.format(repo.name, prx.number, reviewer), m, c0)
         assert log_to_node(repo.log('heads/master')), expected
 
     @pytest.mark.parametrize('separator', [
@@ -1784,15 +1784,15 @@ commits, I need to know how to merge it:
         env.run_crons()
 
         head = repo.commit('heads/master')
-        assert head.message == f"""\
-title
+        assert head.message == textwrap.dedent(f"""\
+        title
 
-first
+        first
 
-closes {repo.name}#{pr.number}
+        closes {repo.name}#{pr.number}
 
-Signed-off-by: {reviewer}
-""", "should not contain the content which follows the thematic break"
+        Signed-off-by: {reviewer}
+        """).strip(), "should not contain the content which follows the thematic break"
 
     def test_pr_message_setex_title(self, repo, env, users, config):
         """ should not break on a proper SETEX-style title """
@@ -1824,21 +1824,21 @@ removed
         env.run_crons()
 
         head = repo.commit('heads/master')
-        assert head.message == f"""\
-title
+        assert head.message == textwrap.dedent(f"""\
+        title
 
-Title
----
-This is some text
+        Title
+        ---
+        This is some text
 
-Title 2
--------
-This is more text
+        Title 2
+        -------
+        This is more text
 
-closes {repo.name}#{pr.number}
+        closes {repo.name}#{pr.number}
 
-Signed-off-by: {reviewer}
-""", "should not break the SETEX titles"
+        Signed-off-by: {reviewer}
+        """).strip(), "should not break the SETEX titles"
 
     def test_rebase_no_edit(self, repo, env, users, config):
         """ Only the merge messages should be de-breaked
@@ -1861,17 +1861,17 @@ Signed-off-by: {reviewer}
         env.run_crons()
 
         head = repo.commit('heads/master')
-        assert head.message == f"""\
-Commit
+        assert head.message == textwrap.dedent(f"""\
+        Commit
 
-first
-***
-second
+        first
+        ***
+        second
 
-closes {repo.name}#{pr.number}
+        closes {repo.name}#{pr.number}
 
-Signed-off-by: {reviewer}
-""", "squashed / rebased messages should not be stripped"
+        Signed-off-by: {reviewer}
+        """).strip(), "squashed / rebased messages should not be stripped"
 
     def test_title_no_edit(self, repo, env, users, config):
         """The first line of a commit message should not be taken in account for
@@ -1903,15 +1903,13 @@ thing: thong
 
 closes {pr_id.display_name}
 
-Signed-off-by: {reviewer}
-"""
+Signed-off-by: {reviewer}"""
         assert repo.commit(staging_head.parents[0]).message == f"""\
 Some: thing
 
 is odd
 
-Part-of: {pr_id.display_name}
-"""
+Part-of: {pr_id.display_name}"""
 
     def test_pr_mergehead(self, repo, env, config):
         """ if the head of the PR is a merge commit and one of the parents is
@@ -1996,7 +1994,7 @@ Part-of: {pr_id.display_name}
         m1 = node('M1')
         reviewer = get_partner(env, users["reviewer"]).formatted_email
         expected = node(
-            'T\n\nTT\n\ncloses {}#{}\n\nSigned-off-by: {}\n'.format(repo.name, prx.number, reviewer),
+            'T\n\nTT\n\ncloses {}#{}\n\nSigned-off-by: {}'.format(repo.name, prx.number, reviewer),
             node('M2', m1),
             node('C1', node('C0', m1), node('B0', m1))
         )
@@ -2072,7 +2070,7 @@ Part-of: {pr_id.display_name}
 
 closes {pr1_id.display_name}
 
-Signed-off-by: {get_partner(env, users["reviewer"]).formatted_email}
+Signed-off-by: {get_partner(env, users["reviewer"]).formatted_email}\
 """
         assert one['commit']['committer']['name'] == a_user['name']
         assert one['commit']['committer']['email'] == a_user['email']
@@ -2103,7 +2101,7 @@ closes {pr2_id.display_name}
 
 Signed-off-by: {get_partner(env, users["reviewer"]).formatted_email}
 Co-authored-by: {a_user['name']} <{a_user['email']}>
-Co-authored-by: {other_user['name']} <{other_user['email']}>
+Co-authored-by: {other_user['name']} <{other_user['email']}>\
 """
         assert repo.read_tree(repo.commit(two['sha'])) == {
             'a': '0',
@@ -2663,12 +2661,12 @@ class TestBatching(object):
         staging = log_to_node(log)
         reviewer = get_partner(env, users["reviewer"]).formatted_email
         p1 = node(
-            'title PR1\n\nbody PR1\n\ncloses {}\n\nSigned-off-by: {}\n'.format(pr1.display_name, reviewer),
+            'title PR1\n\nbody PR1\n\ncloses {}\n\nSigned-off-by: {}'.format(pr1.display_name, reviewer),
             node('initial'),
             node(part_of('commit_PR1_01', pr1), node(part_of('commit_PR1_00', pr1), node('initial')))
         )
         p2 = node(
-            'title PR2\n\nbody PR2\n\ncloses {}\n\nSigned-off-by: {}\n'.format(pr2.display_name, reviewer),
+            'title PR2\n\nbody PR2\n\ncloses {}\n\nSigned-off-by: {}'.format(pr2.display_name, reviewer),
             p1,
             node(part_of('commit_PR2_01', pr2), node(part_of('commit_PR2_00', pr2), p1))
         )
@@ -2703,12 +2701,12 @@ class TestBatching(object):
         reviewer = get_partner(env, users["reviewer"]).formatted_email
 
         p1 = node(
-            'title PR1\n\nbody PR1\n\ncloses {}#{}\n\nSigned-off-by: {}\n'.format(repo.name, pr1.number, reviewer),
+            'title PR1\n\nbody PR1\n\ncloses {}#{}\n\nSigned-off-by: {}'.format(repo.name, pr1.number, reviewer),
             node('initial'),
             node('commit_PR1_01', node('commit_PR1_00', node('initial')))
         )
         p2 = node(
-            'title PR2\n\nbody PR2\n\ncloses {}#{}\n\nSigned-off-by: {}\n'.format(repo.name, pr2.number, reviewer),
+            'title PR2\n\nbody PR2\n\ncloses {}#{}\n\nSigned-off-by: {}'.format(repo.name, pr2.number, reviewer),
             p1,
             node('commit_PR2_01', node('commit_PR2_00', node('initial')))
         )
@@ -2737,8 +2735,8 @@ class TestBatching(object):
 
         staging = log_to_node(log)
         reviewer = get_partner(env, users["reviewer"]).formatted_email
-        expected = node('commit_PR2_00\n\ncloses {}#{}\n\nSigned-off-by: {}\n'.format(repo.name, pr2.number, reviewer),
-             node('commit_PR1_00\n\ncloses {}#{}\n\nSigned-off-by: {}\n'.format(repo.name, pr1.number, reviewer),
+        expected = node('commit_PR2_00\n\ncloses {}#{}\n\nSigned-off-by: {}'.format(repo.name, pr2.number, reviewer),
+             node('commit_PR1_00\n\ncloses {}#{}\n\nSigned-off-by: {}'.format(repo.name, pr1.number, reviewer),
                   node('initial')))
         assert staging == expected
 
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index dbae165c..bee94ef8 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -439,7 +439,7 @@ def test_merge_fail(env, project, repo_a, repo_b, users, config):
         )
     env.run_crons()
 
-    pr2a_id, pr2b_id = s2 = to_pr(env, pr2a) | to_pr(env, pr2b)
+    s2 = to_pr(env, pr2a) | to_pr(env, pr2b)
     st = env['runbot_merge.stagings'].search([])
     assert set(st.batch_ids.prs.ids) == set(s2.ids)
 
@@ -457,14 +457,12 @@ def test_merge_fail(env, project, repo_a, repo_b, users, config):
         c['commit']['message']
         for c in repo_a.log('heads/staging.master')
     ] == [
-        f"""\
-commit_do-b-thing_00
+        """commit_do-b-thing_00
 
-closes {pr2a_id.display_name}
+closes %s
 
-Related: {pr2b_id.display_name}
-Signed-off-by: {reviewer}
-""",
+Related: %s
+Signed-off-by: %s""" % (s2[0].display_name, s2[1].display_name, reviewer),
         'initial'
     ], "dummy commit + squash-merged PR commit + root commit"
 

From 7054c865d78c1c979944e8b9e936dd59c3df3f6d Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 16 Jan 2024 14:54:24 +0100
Subject: [PATCH 059/192] [FIX] *: ngrok tunnel for 3.x

ngrok 3 scrambled some of the tunnel configuration keys. Most notably,
it replaced the ill-named `bind_tls` by an explicit list of http
`schemes`. Although it *removed* `bind_tls` so the fix is necessary
for ngrok to work again (especially as ngrok2 is reaching EOL).

While at it, improve the tunnel setup somewhat:

- remove fixme which we're probably not going to fix after all
- if we spawn ngrok ourselves, keep the handle around so we can
- kill the process we spawned directly instead of looking it up
  somewhat awkwardly
---
 conftest.py | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/conftest.py b/conftest.py
index 3bfb9825..99dec592 100644
--- a/conftest.py
+++ b/conftest.py
@@ -65,7 +65,6 @@ import warnings
 import xmlrpc.client
 from contextlib import closing
 
-import psutil
 import pytest
 import requests
 
@@ -197,6 +196,7 @@ def tunnel(pytestconfig, port):
     if tunnel == '':
         yield f'http://localhost:{port}'
     elif tunnel == 'ngrok':
+        own = None
         web_addr = 'http://localhost:4040/api'
         addr = 'localhost:%d' % port
         # try to find out if ngrok is running, and if it's not attempt
@@ -205,13 +205,9 @@ def tunnel(pytestconfig, port):
             # FIXME: this is for xdist to avoid workers running ngrok at the
             #        exact same time, use lockfile instead
             time.sleep(random.SystemRandom().randint(1, 10))
-            # FIXME: use config file so we can set web_addr to something else
-            #        than localhost:4040 (otherwise we can't disambiguate
-            #        between the ngrok we started and an ngrok started by
-            #        some other user)
             requests.get(web_addr)
         except requests.exceptions.ConnectionError:
-            subprocess.Popen(NGROK_CLI, stdout=subprocess.DEVNULL)
+            own = subprocess.Popen(NGROK_CLI, stdout=subprocess.DEVNULL)
             for _ in range(5):
                 time.sleep(1)
                 with contextlib.suppress(requests.exceptions.ConnectionError):
@@ -223,8 +219,8 @@ def tunnel(pytestconfig, port):
         requests.post(f'{web_addr}/tunnels', json={
             'name': str(port),
             'proto': 'http',
-            'bind_tls': True, # only https
             'addr': addr,
+            'schemes': ['https'],
             'inspect': True,
         }).raise_for_status()
 
@@ -252,17 +248,14 @@ def tunnel(pytestconfig, port):
                     raise TimeoutError("ngrok tunnel deletion failed")
 
                 r = requests.get(f'{web_addr}/tunnels')
+                assert r.ok, f'{r.reason} {r.text}'
                 # there are still tunnels in the list -> bail
-                if r.ok and r.json()['tunnels']:
+                if not own or r.json()['tunnels']:
                     return
 
-                # ngrok is broken or all tunnels have been shut down -> try to
-                # find and kill it (but only if it looks a lot like we started it)
-                for p in psutil.process_iter():
-                    if p.name() == 'ngrok' and p.cmdline() == NGROK_CLI:
-                        p.terminate()
-                        break
-                return
+                # no more tunnels and we started ngrok -> try to kill it
+                own.terminate()
+                own.wait(30)
         else:
             raise TimeoutError("ngrok tunnel creation failed (?)")
     elif tunnel == 'localtunnel':

From 4b9fb513eb0dd2990568c69be8b434c4d8cc949a Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 16 Jan 2024 15:03:45 +0100
Subject: [PATCH 060/192] [IMP] *: make to_pr more resilient to webhook delays

Github delivery delays keep getting worse. Depending on what comes
before `to_pr`, this leads it to fail more often as it runs before the
PR it's looking for was signaled to the mergebot.

In order to mitigate this issue, add a wait loop in `to_pr`, waiting
up to 4 seconds for the PR it's looking for before aborting.

Also replace manual lookups by `to_pr` in every method of
`TestPRUpdate` while at it since it hit a few of the issues. And
remove the xfail test case since it seems unlikely github will change
tack (maybe? could be worth testing to be sure).
---
 mergebot_test_utils/utils.py     | 18 ++++---
 runbot_merge/tests/test_basic.py | 84 +++++++-------------------------
 2 files changed, 30 insertions(+), 72 deletions(-)

diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index 5f61bbb1..22f69ec5 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import itertools
 import re
+import time
 
 from lxml import html
 
@@ -128,12 +129,17 @@ def pr_page(page, pr):
     return html.fromstring(page(f'/{pr.repo.name}/pull/{pr.number}'))
 
 def to_pr(env, pr):
-    pr = env['runbot_merge.pull_requests'].search([
-        ('repository.name', '=', pr.repo.name),
-        ('number', '=', pr.number),
-    ])
-    assert len(pr) == 1, f"Expected to find {pr.repo.name}#{pr.number}, got {pr}."
-    return pr
+    for _ in range(5):
+        pr_id = env['runbot_merge.pull_requests'].search([
+            ('repository.name', '=', pr.repo.name),
+            ('number', '=', pr.number),
+        ])
+        if pr_id:
+            assert len(pr_id) == 1, f"Expected to find {pr.repo.name}#{pr.number}, got {pr_id}."
+            return pr_id
+        time.sleep(1)
+
+    raise TimeoutError(f"Unable to find {pr.repo.name}#{pr.number}")
 
 def part_of(label, pr_id, *, separator='\n\n'):
     """ Adds the "part-of" pseudo-header in the footer.
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 1194283e..bbde05e4 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -2121,10 +2121,8 @@ class TestPRUpdate(object):
 
             c = repo.make_commit(m, 'fist', None, tree={'m': 'c1'})
             prx = repo.make_pr(title='title', body='body', target='master', head=c)
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number),
-        ])
+
+        pr = to_pr(env, prx)
         assert pr.head == c
         # alter & push force PR entirely
         with repo:
@@ -2139,10 +2137,8 @@ class TestPRUpdate(object):
 
             c = repo.make_commit(m, 'fist', None, tree={'m': 'c1'})
             prx = repo.make_pr(title='title', body='body', target='master', head=c)
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number),
-        ])
+
+        pr = to_pr(env, prx)
         with repo:
             prx.close()
         assert pr.state == 'closed'
@@ -2169,10 +2165,7 @@ class TestPRUpdate(object):
             repo.post_status(prx.head, 'success', 'legal/cla')
             repo.post_status(prx.head, 'success', 'ci/runbot')
         env.run_crons()
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number),
-        ])
+        pr = to_pr(env, prx)
         assert pr.head == c
         assert pr.state == 'validated'
 
@@ -2190,10 +2183,8 @@ class TestPRUpdate(object):
             c = repo.make_commit(m, 'fist', None, tree={'m': 'c1'})
             prx = repo.make_pr(title='title', body='body', target='master', head=c)
             prx.post_comment('hansen r+', config['role_reviewer']['token'])
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number),
-        ])
+
+        pr = to_pr(env, prx)
         assert pr.head == c
         assert pr.state == 'approved'
 
@@ -2216,10 +2207,7 @@ class TestPRUpdate(object):
             repo.post_status(prx.head, 'success', 'ci/runbot')
             prx.post_comment('hansen r+', config['role_reviewer']['token'])
         env.run_crons()
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number),
-        ])
+        pr = to_pr(env, prx)
         assert pr.head == c
         assert pr.state == 'ready'
 
@@ -2241,11 +2229,9 @@ class TestPRUpdate(object):
             repo.post_status(prx.head, 'success', 'legal/cla')
             repo.post_status(prx.head, 'success', 'ci/runbot')
             prx.post_comment('hansen r+', config['role_reviewer']['token'])
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number),
-        ])
+
         env.run_crons()
+        pr = to_pr(env, prx)
         assert pr.state == 'ready'
         assert pr.staging_id
 
@@ -2314,11 +2300,8 @@ class TestPRUpdate(object):
             repo.post_status(prx.head, 'success', 'legal/cla')
             repo.post_status(prx.head, 'success', 'ci/runbot')
             prx.post_comment('hansen r+', config['role_reviewer']['token'])
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number),
-        ])
         env.run_crons()
+        pr = to_pr(env, prx)
         assert pr.state == 'ready'
         assert pr.staging_id
 
@@ -2371,10 +2354,7 @@ class TestPRUpdate(object):
 
         with repo:
             prx = repo.make_pr(title='title', body='body', target='master', head=c)
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number),
-        ])
+        pr = to_pr(env, prx)
         assert pr.head == c
         assert pr.state == 'opened'
 
@@ -2407,10 +2387,9 @@ class TestPRUpdate(object):
             repo.post_status(pr.head, 'success', 'legal/cla')
             repo.post_status(pr.head, 'success', 'ci/runbot')
             pr.post_comment('hansen r+', config['role_reviewer']['token'])
-        pr_id = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', pr.number),
-        ])
+
+        env.run_crons()
+        pr_id = to_pr(env, pr)
         env.run_crons('runbot_merge.process_updated_commits')
         assert pr_id.message == 'title\n\nbody'
         assert pr_id.state == 'ready'
@@ -2536,10 +2515,7 @@ Please check and re-approve.
             [c] = repo.make_commits(m, repo.Commit('first', tree={'m': 'm3'}), ref='heads/abranch')
             prx = repo.make_pr(title='title', body='body', target='master', head=c)
         env.run_crons()
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number)
-        ])
+        pr = to_pr(env, prx)
         assert pr.state == 'opened'
         assert pr.head == c
         assert pr.squash
@@ -2575,10 +2551,8 @@ Please check and re-approve.
             repo.post_status(c, 'success', 'ci/runbot')
             prx = repo.make_pr(title='title', body='body', target='master', head=c)
 
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number),
-        ])
+        env.run_crons()
+        pr = to_pr(env, prx)
         assert pr.state == 'validated', \
             "if a PR is created on a CI'd commit, it should be validated immediately"
 
@@ -2589,28 +2563,6 @@ Please check and re-approve.
         assert pr.state == 'validated', \
             "if a PR is reopened and had a CI'd head, it should be validated immediately"
 
-    @pytest.mark.xfail(reason="github doesn't allow reopening force-pushed PRs", strict=True)
-    def test_force_update_closed(self, env, repo):
-        with repo:
-            [m] = repo.make_commits(None, repo.Commit('initial', tree={'m': 'm'}), ref='heads/master')
-
-            [c] = repo.make_commits(m, repo.Commit('first', tree={'m': 'm3'}), ref='heads/abranch')
-            prx = repo.make_pr(title='title', body='body', target='master', head=c)
-        env.run_crons()
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number)
-        ])
-        with repo:
-            prx.close()
-
-        with repo:
-            c2 = repo.make_commit(m, 'xxx', None, tree={'m': 'm4'})
-            repo.update_ref(prx.ref, c2, force=True)
-
-        with repo:
-            prx.open()
-        assert pr.head == c2
 
 class TestBatching(object):
     def _pr(self, repo, prefix, trees, *, target='master', user, reviewer,

From 64d80c276bd7bf6177cc390a5cea7a7cdc6046a9 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 16 Jan 2024 15:07:25 +0100
Subject: [PATCH 061/192] [FIX] *: tests not working with github actual

Add intermediate forks to a pair of tests, because github now (?)
requires being able to write on a branch to create a PR from it, so
the non-collaborator reviewers were not able to create a PR from a
branch created by user.
---
 runbot_merge/tests/test_basic.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index bbde05e4..03dda69c 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -2916,16 +2916,15 @@ class TestReviewing(object):
     def test_self_review_fail(self, env, repo, users, config):
         """ Normal reviewers can't self-review
         """
+        reviewer = config['role_reviewer']['token']
         with repo:
-            m = repo.make_commit(None, 'initial', None, tree={'m': 'm'})
-            m2 = repo.make_commit(m, 'second', None, tree={'m': 'm', 'm2': 'm2'})
-            repo.make_ref('heads/master', m2)
-
-            c1 = repo.make_commit(m, 'first', None, tree={'m': 'c1'})
-            prx = repo.make_pr(title='title', body='body', target='master', head=c1, token=config['role_reviewer']['token'])
+            [m, _] = repo.make_commits(None, Commit('initial', tree={'m': 'm'}), Commit('second', tree={'m2': 'm2'}), ref='heads/master')
+            with repo.fork(token=reviewer) as f:
+                f.make_commits(m, Commit('first', tree={'m': 'c1'}), ref='heads/change')
+            prx = repo.make_pr(title='title', body='body', target='master', head=f'{f.owner}:change', token=reviewer)
             repo.post_status(prx.head, 'success', 'legal/cla')
             repo.post_status(prx.head, 'success', 'ci/runbot')
-            prx.post_comment('hansen r+', config['role_reviewer']['token'])
+            prx.post_comment('hansen r+', reviewer)
         env.run_crons()
 
         assert prx.user == users['reviewer']
@@ -2944,16 +2943,15 @@ class TestReviewing(object):
     def test_self_review_success(self, env, repo, users, config):
         """ Some users are allowed to self-review
         """
+        self_reviewer = config['role_self_reviewer']['token']
         with repo:
-            m = repo.make_commit(None, 'initial', None, tree={'m': 'm'})
-            m2 = repo.make_commit(m, 'second', None, tree={'m': 'm', 'm2': 'm2'})
-            repo.make_ref('heads/master', m2)
-
-            c1 = repo.make_commit(m, 'first', None, tree={'m': 'c1'})
-            prx = repo.make_pr(title='title', body='body', target='master', head=c1, token=config['role_self_reviewer']['token'])
+            [m, _] = repo.make_commits(None, Commit('initial', tree={'m': 'm'}), Commit('second', tree={'m': 'm', 'm2': 'm2'}), ref='heads/master')
+            with repo.fork(token=self_reviewer) as f:
+                f.make_commits(m, Commit('first', tree={'m': 'c1'}), ref='heads/change')
+            prx = repo.make_pr(title='title', body='body', target='master', head=f'{f.owner}:change', token=self_reviewer)
             repo.post_status(prx.head, 'success', 'legal/cla')
             repo.post_status(prx.head, 'success', 'ci/runbot')
-            prx.post_comment('hansen r+', config['role_self_reviewer']['token'])
+            prx.post_comment('hansen r+', self_reviewer)
         env.run_crons()
 
         assert prx.user == users['self_reviewer']

From 95393afde8acdcac3d3e1de26fd7bcadf0ce47f1 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 22 Jan 2024 15:36:37 +0100
Subject: [PATCH 062/192] [FIX] runbot_merge: extraction of authorship info
 during rebase

Turns out I've always been mistaken about the handling of quotes
*inside* shell parameters, apparently they are always consumed by the
shell unless nested so

    --foo="bar"

reaches the underlying program as

    --foo=bar

This means when using subprocess (without shell=True), adding the
quotes leads to mishandling of the parameters (as the subprocess now
has quotes it's not equipped to deal with).

This exact error is made in the `--pretty` parameter of git show,
locally this results in the author name and the committer email being
terminated by double quotes although somehow other layers seem to
exclude those from the end result (I assume `commit-tree` strips the
quotes from the envvars under the assumption that users can mistakenly
quote them or something?

Anyway while it does not seem harmful (so far), better safe than
sorry.
---
 runbot_merge/git.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/git.py b/runbot_merge/git.py
index ead0e6f1..73ad9d73 100644
--- a/runbot_merge/git.py
+++ b/runbot_merge/git.py
@@ -160,7 +160,7 @@ class Repo:
 
         mapping = {}
         for original, tree in zip(commits, new_trees):
-            authorship = check(repo.show('--no-patch', '--pretty="%an%n%ae%n%ai%n%cn%n%ce"', original['sha']))
+            authorship = check(repo.show('--no-patch', '--pretty=%an%n%ae%n%ai%n%cn%n%ce', original['sha']))
             author_name, author_email, author_date, committer_name, committer_email =\
                 authorship.stdout.splitlines()
 

From 3a4fa494f84b3d5f09d3cca1976bb67ff1e4740a Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 12 Feb 2024 10:18:25 +0100
Subject: [PATCH 063/192] [FIX] runbot_merge: incorrectly named endpoint

Method has the same name as its preceding sibling, so it overwrites
it and one of the endpoints is not accessible.
---
 runbot_merge/controllers/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 5d42abe5..535591f1 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -40,7 +40,7 @@ class MergebotController(Controller):
         ]
 
     @route('/runbot_merge/stagings/<int:from_staging>/<int:to_staging>', auth='none', type='json')
-    def prs_for_staging(self, from_staging, to_staging, include_from=True, include_to=True):
+    def prs_for_stagings(self, from_staging, to_staging, include_from=True, include_to=True):
         Stagings = request.env(user=1, context={"active_test": False})['runbot_merge.stagings']
         from_staging = Stagings.browse(from_staging)
         to_staging = Stagings.browse(to_staging)

From 65c303a750ab21906c98a3ee3aa8f77e7f83c984 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 12 Feb 2024 10:18:59 +0100
Subject: [PATCH 064/192] [FIX] runbot_merge: bot info fetch

`r0` is still used afterwards as the response object, so don't
overwrite it when parsing the JSON body.
---
 runbot_merge/models/project.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index 57fc1a33..b686137c 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -74,9 +74,9 @@ class Project(models.Model):
                 _logger.error("Failed to fetch merge bot information for project %s: %s", project.name, r0.text or r0.content)
                 continue
 
-            r0 = r0.json()
-            project.github_name = r0['name'] or r0['login']
-            if email := r0['email']:
+            r = r0.json()
+            project.github_name = r['name'] or r['login']
+            if email := r['email']:
                 project.github_email = email
                 continue
 

From 5d615bd7333313a9c4f94ab2a330653e01237a54 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 12 Feb 2024 10:19:53 +0100
Subject: [PATCH 065/192] [IMP] runbot_merge: logging around webhook body &
 signature

The signature validation code seems correct, but there are validation
failure in production, increase logging around webhook requests to
try and diagnose things better:

- dump the *entire* body to the github_requests logfile
- add the received & computed signatures to the log error
---
 runbot_merge/controllers/__init__.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 535591f1..04b7f8c6 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -96,28 +96,33 @@ class MergebotController(Controller):
 
         secret = env['runbot_merge.repository'].search([
             ('name', '=', repo),
-        ]).project_id.secret
+        ]).project_id.secret.strip()
         if secret:
             signature = 'sha256=' + hmac.new(secret.encode(), req.get_data(), hashlib.sha256).hexdigest()
             if not hmac.compare_digest(signature, req.headers.get('X-Hub-Signature-256', '')):
-                _logger.warning("Ignored hook %s with incorrect signature",
-                                req.headers.get('X-Github-Delivery'))
+                _logger.warning(
+                    "Ignored hook %s with incorrect signature: got %s expected %s",
+                    req.headers.get('X-Github-Delivery'),
+                    req.headers.get('X-Hub-Signature-256'),
+                    signature,
+                )
                 return werkzeug.exceptions.Forbidden()
 
         sentry_sdk.set_context('webhook', request.jsonrequest)
         return c(env, request.jsonrequest)
 
     def _format(self, request):
-        return """<= {r.method} {r.full_path}
+        return """{r.method} {r.full_path}
 {headers}
+
 {body}
-vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
+vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\
 """.format(
             r=request,
             headers='\n'.join(
                 '\t%s: %s' % entry for entry in request.headers.items()
             ),
-            body=utils.shorten(request.get_data(as_text=True).strip(), 400)
+            body=request.get_data(as_text=True),
         )
 
 def handle_pr(env, event):

From de32b54090716b9dd549c1b071d0a2ffeaf4b594 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 23 Feb 2024 13:58:31 +0100
Subject: [PATCH 066/192] [FIX] runbot_merge: error in maintenance, and
 tracking

Before this, we would check if a repository had a name and run
maintenance on it, leading to repeated (but unnoticed until now
because I didn't monitor it) tracebacks as the maintenance cron would
fail to find the local repo then run maintenance on nowhere anyway.

Also augment the repo-finding process to try and get better
information about what it's doing when it fails, rather than failing
completely silently.
---
 runbot_merge/git.py                          | 9 +++++++++
 runbot_merge/models/crons/git_maintenance.py | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/runbot_merge/git.py b/runbot_merge/git.py
index 73ad9d73..771a6733 100644
--- a/runbot_merge/git.py
+++ b/runbot_merge/git.py
@@ -4,6 +4,7 @@ import logging
 import os
 import pathlib
 import resource
+import stat
 import subprocess
 from typing import Optional, TypeVar, Union, Sequence, Tuple, Dict
 
@@ -41,6 +42,14 @@ def get_local(repository, prefix: Optional[str]) -> 'Optional[Repo]':
         repo.config('--add', 'remote.origin.fetch', '^refs/heads/tmp.*')
         repo.config('--add', 'remote.origin.fetch', '^refs/heads/staging.*')
         return repo
+    else:
+        _logger.warning(
+            "Unable to acquire %s: %s",
+            repo_dir,
+            "doesn't exist" if not repo_dir.exists()\
+        else oct(stat.S_IFMT(repo_dir.stat().st_mode))
+        )
+        return None
 
 
 ALWAYS = ('gc.auto=0', 'maintenance.auto=0')
diff --git a/runbot_merge/models/crons/git_maintenance.py b/runbot_merge/models/crons/git_maintenance.py
index 33262d7e..d392d868 100644
--- a/runbot_merge/models/crons/git_maintenance.py
+++ b/runbot_merge/models/crons/git_maintenance.py
@@ -25,7 +25,7 @@ class GC(models.TransientModel):
         # run on all repos with a forwardport target (~ forwardport enabled)
         for repo in self.env['runbot_merge.repository'].search([]):
             repo_git = get_local(repo, prefix=None)
-            if not repo:
+            if not repo_git:
                 continue
 
             _gc.info('Running maintenance on %s', repo.name)

From bcf6074153d338b895ead938f5b9891d66059d40 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 26 Feb 2024 09:56:25 +0100
Subject: [PATCH 067/192] [FIX] runbot_merge: maintenance gc command

`gc --prune` can not take a *separate* parameter, it has to be part of
the same arg (the `=` is not optional), otherwise the `gc` call blows
up.

So use the positional form of the git command to generate the correct
invocation, Python-level `foo=bar` generates a split-style option in
two args which does not please git.
---
 runbot_merge/models/crons/git_maintenance.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/models/crons/git_maintenance.py b/runbot_merge/models/crons/git_maintenance.py
index d392d868..2b0e2182 100644
--- a/runbot_merge/models/crons/git_maintenance.py
+++ b/runbot_merge/models/crons/git_maintenance.py
@@ -32,6 +32,6 @@ class GC(models.TransientModel):
             r = repo_git\
                 .stdout(True)\
                 .with_config(stderr=subprocess.STDOUT, text=True, check=False)\
-                .gc(aggressive=True, prune='now')
+                .gc('--prune=now', aggressive=True)
             if r.returncode:
                 _gc.warning("Maintenance failure (status=%d):\n%s", r.returncode, r.stdout)

From 721b76903909149bbceeef95150d8ca9a91f6ca6 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 26 Feb 2024 10:11:53 +0100
Subject: [PATCH 068/192] [IMP] runbot_merge: handling of signatures

- correctly handle projects without a secret set, we don't want the
  requests to blow up by trying to `strip()` a `False` or `None`, that
  is dumb, who would do that?
- provide better reporting on signature mismatch: which repo we tried
  to access, and the full list of headers
- log when there was no signature matching, either because there was
  no signature in the request and no secret on the project, or because
  the request is signed but no secret is configured on the repo
---
 runbot_merge/controllers/__init__.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 04b7f8c6..35c4e542 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -96,17 +96,23 @@ class MergebotController(Controller):
 
         secret = env['runbot_merge.repository'].search([
             ('name', '=', repo),
-        ]).project_id.secret.strip()
-        if secret:
-            signature = 'sha256=' + hmac.new(secret.encode(), req.get_data(), hashlib.sha256).hexdigest()
+        ]).project_id.secret
+        if secret and secret.strip():
+            signature = 'sha256=' + hmac.new(secret.strip().encode(), req.get_data(), hashlib.sha256).hexdigest()
             if not hmac.compare_digest(signature, req.headers.get('X-Hub-Signature-256', '')):
                 _logger.warning(
-                    "Ignored hook %s with incorrect signature: got %s expected %s",
+                    "Ignored hook %s with incorrect signature on %s: got %s expected %s, in:\n%s",
                     req.headers.get('X-Github-Delivery'),
+                    repo,
                     req.headers.get('X-Hub-Signature-256'),
                     signature,
+                    req.headers,
                 )
                 return werkzeug.exceptions.Forbidden()
+        elif req.headers.get('X-Hub-Signature-256'):
+            _logger.info("No secret for %s but received a signature in:\n%s", repo, req.headers)
+        else:
+            _logger.info("No secret or signature for %s", repo)
 
         sentry_sdk.set_context('webhook', request.jsonrequest)
         return c(env, request.jsonrequest)

From 327500bc8356d242be655af1ae07f916c80efed6 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 12 Mar 2024 12:17:30 +0100
Subject: [PATCH 069/192] [FIX] runbot_merge: don't notify on closing unknown
 PRs

If an untracked PR is closed, especially on an inactive or untracked
branch, the closer (or author) almost certainly don't care to receive
3 different notifications on the subject.

The fix requires a schema change in order to track that we're fetching
the PR due to a `closed` event, as in other cases we may still want to
notify the user that we received the request (and it just happened to
resolve to a closed PR).

Fixes #857
---
 runbot_merge/controllers/__init__.py |  2 +-
 runbot_merge/models/pull_requests.py | 44 ++++++++++++------
 runbot_merge/tests/test_basic.py     | 68 ++++++++++++++++++++++++++++
 3 files changed, 98 insertions(+), 16 deletions(-)

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 35c4e542..afbe92d6 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -257,7 +257,7 @@ def handle_pr(env, event):
         pr_obj = env['runbot_merge.pull_requests']._from_gh(pr)
         return "Tracking PR as {}".format(pr_obj.id)
 
-    pr_obj = env['runbot_merge.pull_requests']._get_or_schedule(r, pr['number'])
+    pr_obj = env['runbot_merge.pull_requests']._get_or_schedule(r, pr['number'], closing=event['action'] == 'closed')
     if not pr_obj:
         _logger.info("webhook %s on unknown PR %s#%s, scheduled fetch", event['action'], repo.name, pr['number'])
         return "Unknown PR {}:{}, scheduling fetch".format(repo.name, pr['number'])
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 96581736..1b7a120b 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -96,28 +96,30 @@ All substitutions are tentatively applied sequentially to the input.
             self._cr, 'runbot_merge_unique_repo', self._table, ['name'])
         return res
 
-    def _load_pr(self, number):
+    def _load_pr(self, number, *, closing=False):
         gh = self.github()
 
         # fetch PR object and handle as *opened*
         issue, pr = gh.pr(number)
 
+        repo_name = pr['base']['repo']['full_name']
         if not self.project_id._has_branch(pr['base']['ref']):
-            _logger.info("Tasked with loading PR %d for un-managed branch %s:%s, ignoring",
-                         number, self.name, pr['base']['ref'])
-            self.env.ref('runbot_merge.pr.load.unmanaged')._send(
-                repository=self,
-                pull_request=number,
-                format_args = {
-                    'pr': pr,
-                    'repository': self,
-                },
-            )
+            _logger.info("Tasked with loading %s PR %s#%d for un-managed branch %s:%s, ignoring",
+                         pr['state'], repo_name, number, self.name, pr['base']['ref'])
+            if not closing:
+                self.env.ref('runbot_merge.pr.load.unmanaged')._send(
+                    repository=self,
+                    pull_request=number,
+                    format_args = {
+                        'pr': pr,
+                        'repository': self,
+                    },
+                )
             return
 
         # if the PR is already loaded, force sync a few attributes
         pr_id = self.env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', pr['base']['repo']['full_name']),
+            ('repository.name', '=', repo_name),
             ('number', '=', number),
         ])
         if pr_id:
@@ -143,6 +145,11 @@ All substitutions are tentatively applied sequentially to the input.
                     'pull_request': pr,
                     'sender': {'login': self.project_id.github_prefix}
                 }) + '. '
+            if pr_id.state != 'closed' and pr['state'] == 'closed':
+                # don't go through controller because try_closing does weird things
+                # for safety / race condition reasons which ends up committing
+                # and breaks everything
+                pr_id.state = 'closed'
             self.env['runbot_merge.pull_requests.feedback'].create({
                 'repository': pr_id.repository.id,
                 'pull_request': number,
@@ -150,6 +157,11 @@ All substitutions are tentatively applied sequentially to the input.
             })
             return
 
+        # special case for closed PRs, just ignore all events and skip feedback
+        if closing:
+            self.env['runbot_merge.pull_requests']._from_gh(pr)
+            return
+
         sender = {'login': self.project_id.github_prefix}
         # init the PR to the null commit so we can later synchronise it back
         # back to the "proper" head while resetting reviews
@@ -199,7 +211,7 @@ All substitutions are tentatively applied sequentially to the input.
             'sender': sender,
         })
         pr_id = self.env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', pr['base']['repo']['full_name']),
+            ('repository.name', '=', repo_name),
             ('number', '=', number),
         ])
         if pr['state'] == 'closed':
@@ -520,7 +532,7 @@ class PullRequests(models.Model):
         for r in self:
             r.batch_id = r.batch_ids.filtered(lambda b: b.active)[:1]
 
-    def _get_or_schedule(self, repo_name, number, target=None):
+    def _get_or_schedule(self, repo_name, number, *, target=None, closing=False):
         repo = self.env['runbot_merge.repository'].search([('name', '=', repo_name)])
         if not repo:
             return
@@ -546,6 +558,7 @@ class PullRequests(models.Model):
         Fetch.create({
             'repository': repo.id,
             'number': number,
+            'closing': closing,
         })
 
     def _parse_command(self, commandstring):
@@ -1823,6 +1836,7 @@ class FetchJob(models.Model):
     active = fields.Boolean(default=True)
     repository = fields.Many2one('runbot_merge.repository', required=True)
     number = fields.Integer(required=True, group_operator=None)
+    closing = fields.Boolean(default=False)
 
     def _check(self, commit=False):
         """
@@ -1835,7 +1849,7 @@ class FetchJob(models.Model):
 
             self.env.cr.execute("SAVEPOINT runbot_merge_before_fetch")
             try:
-                f.repository._load_pr(f.number)
+                f.repository._load_pr(f.number, closing=f.closing)
             except Exception:
                 self.env.cr.execute("ROLLBACK TO SAVEPOINT runbot_merge_before_fetch")
                 _logger.exception("Failed to load pr %s, skipping it", f.number)
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 03dda69c..1244bda2 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -3218,6 +3218,74 @@ class TestUnknownPR:
                             "as I didn't see previous commands."),
         ]
 
+    def test_close_unknown_unmanaged(self, env, repo, users, config):
+        """If an "unknown PR" is *closed*, it should be saved as closed but not
+        commented on, because that's unnecessary spam.
+        """
+        with repo:
+            m, _ = repo.make_commits(
+                None,
+                Commit('initial', tree={'m': 'm'}),
+                Commit('second', tree={'m2': 'm2'}),
+                ref='heads/master')
+
+            [c1] = repo.make_commits(m, Commit('first', tree={'m': 'c1'}))
+            pr = repo.make_pr(title='title', body='body', target='master', head=c1)
+        env.run_crons()
+        assert pr.comments == [seen(env, pr, users)]
+
+        to_pr(env, pr).unlink()
+        env['runbot_merge.commit'].search([('sha', '=', pr.head)]).unlink()
+
+        with repo:
+            pr.close()
+
+        Fetch = env['runbot_merge.fetch_job']
+        fetches = Fetch.search([('repository', '=', repo.name), ('number', '=', pr.number)])
+        assert len(fetches) == 1, f"expected one fetch for {pr.number}, found {len(fetches)}"
+
+        env.run_crons('runbot_merge.fetch_prs_cron')
+        env.run_crons()
+        assert not Fetch.search([('repository', '=', repo.name), ('number', '=', pr.number)])
+
+        assert to_pr(env, pr).state == 'closed'
+        assert pr.comments == [seen(env, pr, users)]
+
+
+    def test_close_unknown_disabled(self, env, repo, users, config):
+        """If an "unknown PR" on an disabled branch is *closed*, it should be
+        saved as closed but not commented on, because that's unnecessary spam.
+        """
+        with repo:
+            m, _ = repo.make_commits(
+                None,
+                Commit('initial', tree={'m': 'm'}),
+                Commit('second', tree={'m2': 'm2'}),
+                ref='heads/master')
+
+            [c1] = repo.make_commits(m, Commit('first', tree={'m': 'c1'}))
+            pr = repo.make_pr(title='title', body='body', target='master', head=c1)
+        env.run_crons()
+        assert pr.comments == [seen(env, pr, users)]
+
+        to_pr(env, pr).unlink()
+        env['runbot_merge.commit'].search([('sha', '=', pr.head)]).unlink()
+        env['runbot_merge.branch'].search([('name', '=', 'master')]).active = False
+
+        with repo:
+            pr.close()
+
+        Fetch = env['runbot_merge.fetch_job']
+        fetches = Fetch.search([('repository', '=', repo.name), ('number', '=', pr.number)])
+        assert len(fetches) == 1, f"expected one fetch for {pr.number}, found {len(fetches)}"
+
+        env.run_crons('runbot_merge.fetch_prs_cron')
+        env.run_crons()
+        assert not Fetch.search([('repository', '=', repo.name), ('number', '=', pr.number)])
+
+        assert to_pr(env, pr).state == 'closed'
+        assert pr.comments == [seen(env, pr, users)]
+
     def test_rplus_unmanaged(self, env, repo, users, config):
         """ r+ on an unmanaged target should notify about
         """

From 953bf86044ac76c78a6a9c6375f60e31677ad220 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 12 Mar 2024 14:57:50 +0100
Subject: [PATCH 070/192] [FIX] forwardport: don't notify of detached child on
 merged parents

The notification is both noise and confusing: we're telling the
author (and reviewer, and anyone else subscribed) that they need to
merge a merged PR.

Fixes #855
---
 forwardport/models/project.py     | 13 +++++-----
 forwardport/tests/test_updates.py | 42 +++++++++++++++++++++++--------
 2 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index ff977628..405ca8d2 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -363,12 +363,13 @@ class PullRequests(models.Model):
                     token_field='fp_github_token',
                     format_args={'pr': p},
                 )
-                self.env.ref('runbot_merge.forwardport.update.parent')._send(
-                    repository=parent.repository,
-                    pull_request=parent.number,
-                    token_field='fp_github_token',
-                    format_args={'pr': parent, 'child': p},
-                )
+                if parent.state not in ('closed', 'merged'):
+                    self.env.ref('runbot_merge.forwardport.update.parent')._send(
+                        repository=parent.repository,
+                        pull_request=parent.number,
+                        token_field='fp_github_token',
+                        format_args={'pr': parent, 'child': p},
+                    )
         for p in closed_fp.filtered(lambda p: p.state != 'closed'):
             self.env.ref('runbot_merge.forwardport.reopen.detached')._send(
                 repository=p.repository,
diff --git a/forwardport/tests/test_updates.py b/forwardport/tests/test_updates.py
index c83afde0..1f5269df 100644
--- a/forwardport/tests/test_updates.py
+++ b/forwardport/tests/test_updates.py
@@ -4,10 +4,13 @@ initial merge has succeeded (and forward-porting has started)
 """
 import re
 
+import pytest
+
 from utils import seen, re_matches, Commit, make_basic, to_pr
 
 
-def test_update_pr(env, config, make_repo, users):
+@pytest.mark.parametrize("merge_parent", [False, True])
+def test_update_pr(env, config, make_repo, users, merge_parent) -> None:
     """ Even for successful cherrypicks, it's possible that e.g. CI doesn't
     pass or the reviewer finds out they need to update the code.
 
@@ -134,7 +137,18 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
         prod.post_status(pr2_id.head, 'success', 'ci/runbot')
         prod.post_status(pr2_id.head, 'success', 'legal/cla')
     env.run_crons()
-    
+
+    pr2 = prod.get_pr(pr2_id.number)
+    if merge_parent:
+        with prod:
+            pr2.post_comment('hansen r+', config['role_reviewer']['token'])
+        env.run_crons()
+        with prod:
+            prod.post_status('staging.c', 'success', 'ci/runbot')
+            prod.post_status('staging.c', 'success', 'legal/cla')
+        env.run_crons()
+        assert pr2_id.state == 'merged'
+
     _0, _1, _2, pr3_id = env['runbot_merge.pull_requests'].search([], order='number')
     assert pr3_id.parent_id == pr2_id
     # don't bother updating heads (?)
@@ -160,21 +174,29 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
                         f"(via @{pr3_id.repository.project_id.github_prefix})"
         )
     ]
-    pr2 = prod.get_pr(pr2_id.number)
-    assert pr2.comments == [
+
+    assert pr2.comments[:2] == [
         seen(env, pr2, users),
         (users['user'], """\
 This PR targets c and is part of the forward-port chain. Further PRs will be created up to d.
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 """),
-        (users['user'], f"@{users['user']} @{users['reviewer']} child PR "
-                        f"{pr3_id.display_name} was modified / updated and has "
-                        f"become a normal PR. This PR (and any of its parents) "
-                        f"will need to be merged independently as approvals "
-                        f"won't cross."),
     ]
-    
+
+    if merge_parent:
+        assert pr2.comments[2:] == [
+            (users['reviewer'], "hansen r+"),
+        ]
+    else:
+        assert pr2.comments[2:] == [
+            (users['user'], f"@{users['user']} @{users['reviewer']} child PR "
+                            f"{pr3_id.display_name} was modified / updated and has "
+                            f"become a normal PR. This PR (and any of its parents) "
+                            f"will need to be merged independently as approvals "
+                            f"won't cross."),
+        ]
+
 
 def test_update_merged(env, make_repo, config, users):
     """ Strange things happen when an FP gets closed / merged but then its

From 5024c2e27b1df702b9e8014fb81c873254281e81 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 19 Mar 2024 11:46:36 +0100
Subject: [PATCH 071/192] [FIX] forwardport: suppress warning when closing
 unmanaged PR

Continuation of 327500bc8356d242be655af1ae07f916c80efed6 for an other
edge case of closing a PR to a detached branch with a merged
descendant. The mergebot would:

- warn on the parent about it being detached due to being closed
- then warn on the child about it being detached due to the parent
  being closed (despite it being merged already)
- then warn the parent *again* due to the child being detached

At least some of those messages were still produced by the test case,
stop them.

Issue was noticed on odoo/odoo#145969 and odoo/odoo#145984 due to 16.2
being deactivated.
---
 forwardport/models/project.py    |  1 +
 forwardport/tests/test_simple.py | 80 +++++++++++++++++++++++++++++++-
 2 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 405ca8d2..13def2bf 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -332,6 +332,7 @@ class PullRequests(models.Model):
         with_parents = {
             p: p.parent_id
             for p in self
+            if p.state not in ('merged', 'closed')
             if p.parent_id
         }
         closed_fp = self.filtered(lambda p: p.state == 'closed' and p.source_id)
diff --git a/forwardport/tests/test_simple.py b/forwardport/tests/test_simple.py
index 5f5e21b8..6d398f64 100644
--- a/forwardport/tests/test_simple.py
+++ b/forwardport/tests/test_simple.py
@@ -6,7 +6,7 @@ from datetime import datetime, timedelta
 
 import pytest
 
-from utils import seen, Commit, make_basic, REF_PATTERN, MESSAGE_TEMPLATE, validate_all, part_of
+from utils import seen, Commit, make_basic, REF_PATTERN, MESSAGE_TEMPLATE, validate_all, part_of, to_pr
 
 FMT = '%Y-%m-%d %H:%M:%S'
 FAKE_PREV_WEEK = (datetime.now() + timedelta(days=1)).strftime(FMT)
@@ -884,6 +884,84 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
             ),
         )
 
+    def test_close_disabled(self, env, make_repo, users, config):
+        """ If an fwport's target is disabled and its branch is closed, it
+        should not be notified (multiple times), also its descendant should not
+        be nodified if already merged, also there should not be recursive
+        notifications (odoo/odoo#145969, odoo/odoo#145984)
+        """
+        repo, _ = make_basic(env, config, make_repo)
+        env['runbot_merge.repository'].search([]).required_statuses = 'default'
+        # prep: merge PR, create two forward ports
+        with repo:
+            [c1] = repo.make_commits('a', Commit('first', tree={'m': 'c1'}))
+            pr1 = repo.make_pr(title='title', body='body', target='a', head=c1)
+            pr1.post_comment('hansen r+', config['role_reviewer']['token'])
+            repo.post_status(c1, 'success')
+        env.run_crons()
+
+        pr1_id = to_pr(env, pr1)
+        assert pr1_id.state == 'ready', pr1_id.blocked
+
+        with repo:
+            repo.post_status('staging.a', 'success')
+        env.run_crons()
+
+        pr1_id_, pr2_id = env['runbot_merge.pull_requests'].search([], order='number')
+        assert pr1_id_ == pr1_id
+        with repo:
+            repo.post_status(pr2_id.head, 'success')
+        env.run_crons()
+
+        _, _, pr3_id = env['runbot_merge.pull_requests'].search([], order='number')
+
+        # disable second branch
+        pr2_id.target.active = False
+        env.run_crons()
+
+        pr2 = repo.get_pr(pr2_id.number)
+        assert pr2.comments == [
+            seen(env, pr2, users),
+            (users['user'], "This PR targets b and is part of the forward-port chain. "
+                           "Further PRs will be created up to c.\n\n"
+                           "More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n"),
+            (users['user'], "@{user} @{reviewer} the target branch 'b' has been disabled, you may want to close this PR.".format_map(
+                users
+            )),
+        ]
+        pr3 = repo.get_pr(pr3_id.number)
+        assert pr3.comments == [
+            seen(env, pr3, users),
+            (users['user'], """\
+@{user} @{reviewer} this PR targets c and is the last of the forward-port chain containing:
+* {pr2_id.display_name}
+
+To merge the full chain, use
+> @herbert r+
+
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+""".format(pr2_id=pr2_id, **users)),
+        ]
+
+        # some time later, notice PR3 is open and merge it
+        with repo:
+            pr3.post_comment('hansen r+', config['role_reviewer']['token'])
+            repo.post_status(pr3.head, 'success')
+        env.run_crons()
+        with repo:
+            repo.post_status('staging.c', 'success')
+        env.run_crons()
+
+        assert pr3_id.status == 'success'
+
+        # even later, notice PR2 is still open but not mergeable anymore
+        with repo:
+            pr2.close()
+        env.run_crons()
+
+        assert pr2.comments[3:] == []
+        assert pr3.comments[2:] == [(users['reviewer'], "hansen r+")]
+
 class TestBranchDeletion:
     def test_delete_normal(self, env, config, make_repo):
         """ Regular PRs should get their branch deleted as long as they're

From 9f22305903c54315c1bd24d5a7f2ba648fd317eb Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 19 Mar 2024 12:24:43 +0100
Subject: [PATCH 072/192] [IMP] runbot_merge: view warnings around ACLs

Eventually we might want to add a proper "sensitive" flag on overrides
and compute the flag based on that. For now just check for
`ci/security`.
---
 runbot_merge/models/res_partner.py |  6 ++++++
 runbot_merge/views/res_partner.xml | 15 +++++++++++++--
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/models/res_partner.py b/runbot_merge/models/res_partner.py
index d627b2f4..8815e133 100644
--- a/runbot_merge/models/res_partner.py
+++ b/runbot_merge/models/res_partner.py
@@ -19,6 +19,7 @@ class Partner(models.Model):
     formatted_email = fields.Char(string="commit email", compute='_rfc5322_formatted')
     review_rights = fields.One2many('res.partner.review', 'partner_id')
     override_rights = fields.Many2many('res.partner.override')
+    override_sensitive = fields.Boolean(compute="_compute_sensitive_overrides")
 
     def _auto_init(self):
         res = super(Partner, self)._auto_init()
@@ -45,6 +46,11 @@ class Partner(models.Model):
             p.email = gh.user(p.github_login)['email'] or False
         return False
 
+    @api.depends("override_rights.context")
+    def _compute_sensitive_overrides(self):
+        for p in self:
+            p.override_sensitive = any(o.context == 'ci/security' for o in p.override_rights)
+
 class PartnerMerge(models.TransientModel):
     _inherit = 'base.partner.merge.automatic.wizard'
 
diff --git a/runbot_merge/views/res_partner.xml b/runbot_merge/views/res_partner.xml
index 507e5a32..bad17f8f 100644
--- a/runbot_merge/views/res_partner.xml
+++ b/runbot_merge/views/res_partner.xml
@@ -50,6 +50,7 @@
             </xpath>
             <xpath expr="//notebook" position="inside">
                 <page string="Mergebot" groups="runbot_merge.group_admin">
+                    <field name="override_sensitive" invisible="1"/>
                     <group>
                         <group>
                             <field name="github_login"/>
@@ -57,7 +58,12 @@
                     </group>
                     <group>
                         <group colspan="4" string="Review Rights">
-                            <field name="review_rights" nolabel="1">
+                            <div colspan="4" class="alert alert-warning" role="alert" attrs="{'invisible': [('review_rights', '=', [])]}">
+                                Review access requires successfully following
+                                the Code Review (QDP) and Security (DLE)
+                                trainings. Please check before giving r+ access.
+                            </div>
+                            <field colspan="4" name="review_rights" nolabel="1">
                                 <tree string="Review ACLs" editable="bottom">
                                     <field name="repository_id"/>
                                     <field name="review"/>
@@ -66,7 +72,12 @@
                             </field>
                         </group>
                         <group colspan="4">
-                            <field name="override_rights" widget="many2many_tags"/>
+                            <div colspan="4" class="alert alert-danger" role="alert" attrs="{'invisible': [('override_sensitive', '=', False)]}">
+                                Security Override <b>REQUIRES</b> successfully
+                                following the Security training. Please ask DLE
+                                before granting access.
+                            </div>
+                            <field colspan="4" name="override_rights" widget="many2many_tags"/>
                         </group>
                     </group>
                     <group>

From a8e4d6dfee682e6ff926b7b0e03b63f5c4aea583 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 7 Feb 2024 10:48:28 +0100
Subject: [PATCH 073/192] [IMP] runbot_merge: don't select content when locking
 rows

It might not be a huge amount of extra work since we're never actually
retrieving the rows, but it still seems completely unnecessary.

Sadly we can't do something cleaner like an aggregation, because
aggregating requires moving the locking query to a subquery, and
experimentally that seems slower than just ignoring / discarding the
result set.
---
 runbot_merge/models/project_freeze/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py
index c5b3312e..000b8835 100644
--- a/runbot_merge/models/project_freeze/__init__.py
+++ b/runbot_merge/models/project_freeze/__init__.py
@@ -184,7 +184,7 @@ class FreezeWizard(models.Model):
         # we don't want to run concurrently to the crons above, though we
         # don't need to prevent read access to them
         self.env.cr.execute(
-            'SELECT * FROM ir_cron WHERE id =ANY(%s) FOR SHARE NOWAIT',
+            'SELECT FROM ir_cron WHERE id =ANY(%s) FOR SHARE NOWAIT',
             [conflict_crons.ids]
         )
 
@@ -195,7 +195,7 @@ class FreezeWizard(models.Model):
         master, rest = project_id.branch_ids[0], project_id.branch_ids[1:]
         if self.bump_pr_ids and master.active_staging_id:
             self.env.cr.execute(
-                'SELECT * FROM runbot_merge_stagings WHERE id = %s FOR UPDATE NOWAIT',
+                'SELECT FROM runbot_merge_stagings WHERE id = %s FOR UPDATE NOWAIT',
                 [master.active_staging_id.id]
             )
 

From b177361f20ee3c2c9c866c0945ecb38b18bf677e Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 15 Dec 2023 10:38:08 +0100
Subject: [PATCH 074/192] [IMP] forwardport: locking during creation of fw
 batch

Not sure why I didn't think about it previously (in #357), but it
would make sense for the entire batch creation to be atomic and thus
under the same lock.

The commit during forward porting also makes a lot less sense: it was
a failed early attempt at resolving the problem by hoping we'd win the
race with the webhook (commit before the webhook hit). By locking the
PRs table to update, we actually resolved it.

But since all that happens then is a few updates and then a commit by
the cron itself (it commits per batch), it's probably good enough to
leave the entire thing under the same lock. This means we lock out
other interactions a bit longer, but since the span is still just the
forward port of a single batch it should not be too much of an issue
outside of post-freeze recovery thingie.
---
 forwardport/models/project.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 13def2bf..5aaf3a20 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -810,6 +810,7 @@ class PullRequests(models.Model):
         # one of the PRs in the batch fails is huge problem, though this loop
         # only concerns itself with the creation of the followup objects so...
         new_batch = self.browse(())
+        self.env.cr.execute('LOCK runbot_merge_pull_requests IN SHARE MODE')
         for pr in self:
             owner, _ = pr.repository.fp_remote_target.split('/', 1)
             source = pr.source_id or pr
@@ -821,7 +822,6 @@ class PullRequests(models.Model):
             )
 
             title, body = re.match(r'(?P<title>[^\n]+)\n*(?P<body>.*)', message, flags=re.DOTALL).groups()
-            self.env.cr.execute('LOCK runbot_merge_pull_requests IN SHARE MODE')
             r = gh.post(f'https://api.github.com/repos/{pr.repository.name}/pulls', json={
                 'base': target.name,
                 'head': f'{owner}:{new_branch}',
@@ -866,10 +866,6 @@ class PullRequests(models.Model):
                     token_field='fp_github_token',
                     format_args={'source': source, 'pr': pr, 'new': new_pr, 'footer': footer},
                 )
-            # not great but we probably want to avoid the risk of the webhook
-            # creating the PR from under us. There's still a "hole" between
-            # the POST being executed on gh and the commit but...
-            self.env.cr.commit()
 
         for pr, new_pr in zip(self, new_batch):
             (h, out, err, hh) = conflicts.get(pr) or (None, None, None, None)

From f4889ec8cfa4a1c689d1b719d46060aeaa51fc8d Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 22 Mar 2024 14:24:13 +0100
Subject: [PATCH 075/192] [ADD] runbot_merge: ad-hoc ACL tracking to
 res.partner

Sadly m2ms don't support tracking, so add a bunch of ad-hoc tracking
to the override rights in order to know who, what, when at least.

Do the same for the review rights although maybe tracking works for
those.
---
 runbot_merge/models/res_partner.py | 104 ++++++++++++++++++++++++++++-
 1 file changed, 102 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/models/res_partner.py b/runbot_merge/models/res_partner.py
index 8815e133..f6c78d6f 100644
--- a/runbot_merge/models/res_partner.py
+++ b/runbot_merge/models/res_partner.py
@@ -1,7 +1,10 @@
 import random
 from email.utils import parseaddr
 
-from odoo import fields, models, tools, api
+from markupsafe import Markup, escape
+
+import odoo.tools
+from odoo import fields, models, tools, api, Command
 
 from .. import github
 
@@ -11,7 +14,8 @@ class CIText(fields.Char):
     column_cast_from = ('varchar', 'text')
 
 class Partner(models.Model):
-    _inherit = 'res.partner'
+    _name = 'res.partner'
+    _inherit = ['res.partner', 'mail.thread']
 
     email = fields.Char(index=True)
     github_login = CIText()
@@ -51,6 +55,66 @@ class Partner(models.Model):
         for p in self:
             p.override_sensitive = any(o.context == 'ci/security' for o in p.override_rights)
 
+    def write(self, vals):
+        created = []
+        updated = {}
+        deleted = set()
+        for cmd, id, values in vals.get('review_rights', []):
+            if cmd == Command.DELETE:
+                deleted.add(id)
+            elif cmd == Command.CREATE:
+                # 'repository_id': 3, 'review': True, 'self_review': False
+                created.append(values)
+            elif cmd == Command.UPDATE:
+                updated[id] = values
+            #  could also be LINK for records which are not touched but we don't care
+
+        new_rights = None
+        if r := vals.get('override_rights'):
+            # only handle reset (for now?) even though technically e.g. 0 works
+            # the web client doesn't seem to use it (?)
+            if r[0][0] == 6:
+                new_rights = self.env['res.partner.override'].browse(r[0][2])
+
+        Repo = self.env['runbot_merge.repository'].browse
+        for p in self:
+            msgs = []
+            if ds := p.review_rights.filtered(lambda r: r.id in deleted):
+                msgs.append("removed review rights on {}\n".format(
+                    ', '.join(ds.mapped('repository_id.name'))
+                ))
+            if us := p.review_rights.filtered(lambda r: r.id in updated):
+                msgs.extend(
+                    "updated review rights on {}: {}\n".format(
+                        u.repository_id.name,
+                        ', '.join(
+                            f'allowed {f}' if v else f'forbid {f}'
+                            for f in ['review', 'self_review']
+                            if (v := updated[u.id].get(f)) is not None
+                        )
+                    )
+                    for u in us
+                )
+            msgs.extend(
+                'added review rights on {}: {}\n'.format(
+                    Repo(c['repository_id']).name,
+                    ', '.join(filter(c.get, ['review', 'self_review'])),
+                )
+                for c in created
+            )
+            if new_rights is not None:
+                for r in p.override_rights - new_rights:
+                    msgs.append(f"removed override rights for {r.context!r} on {r.repository_id.name}")
+                for r in new_rights - p.override_rights:
+                    msgs.append(f"added override rights for {r.context!r} on {r.repository_id.name}")
+            if msgs:
+                p._message_log(body=Markup('<ul>{}</ul>').format(Markup().join(
+                    map(Markup('<li>{}</li>').format, reversed(msgs))
+                )))
+
+        return super().write(vals)
+
+
 class PartnerMerge(models.TransientModel):
     _inherit = 'base.partner.merge.automatic.wizard'
 
@@ -109,6 +173,42 @@ class OverrideRights(models.Model):
             ['context', 'coalesce(repository_id, 0)']
         )
 
+    @api.model_create_multi
+    def create(self, vals_list):
+        for partner, contexts in odoo.tools.groupby((
+            (partner_id, vals['context'], vals['repository_id'])
+            for vals in vals_list
+            # partner_ids is of the form [Command.set(ids)
+            for partner_id in vals.get('partner_ids', [(None, None, [])])[0][2]
+        ), lambda p: p[0]):
+            partner = self.env['res.partner'].browse(partner)
+            for _, context, repository in contexts:
+                repository = self.env['runbot_merge.repository'].browse(repository)
+                partner._message_log(body=f"added override rights for {context!r} on {repository.name}")
+
+        return super().create(vals_list)
+
+    def write(self, vals):
+        new = None
+        if pids := vals.get('partner_ids'):
+            new = self.env['res.partner'].browse(pids[0][2])
+        if new is not None:
+            for o in self:
+                added = new - o.partner_ids
+                removed = o.partner_ids - new
+                for p in added:
+                    p._message_log(body=f"added override rights for {o.context!r} on {o.repository_id.name}")
+                for r in removed:
+                    r._message_log(body=f"removed override rights for {o.context!r} on {o.repository_id.name}")
+
+        return super().write(vals)
+
+    def unlink(self):
+        for o in self:
+            for p in o.partner_ids:
+                p._message_log(body=f"removed override rights for {o.context!r} on {o.repository_id.name}")
+        return super().unlink()
+
     @api.model
     def name_search(self, name='', args=None, operator='ilike', limit=100):
         return self.search((args or []) + [

From 0e71f85802bcf17719c3c3c6baf99d301d76e89f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 16 May 2024 10:36:34 +0200
Subject: [PATCH 076/192] [ADD] forwardport: some typing to conftest

It's not amazing, because cross-typing between the different conftests
and utility modules doesn't really work smoothly. So not entirely
convinced it's great.

While at it, inline a few `x.json()` local aliases when the jsons are
used in exclusive locations e.g. usually an assertion message on one
side and a result / process on the other.
---
 conftest.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/conftest.py b/conftest.py
index 99dec592..090ebb4b 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,4 +1,7 @@
-# -*- coding: utf-8 -*-
+from __future__ import annotations
+
+from typing import Optional
+
 """
 Configuration:
 
@@ -614,7 +617,7 @@ class Repo:
         assert res['object']['type'] == 'commit'
         return res['object']['sha']
 
-    def commit(self, ref):
+    def commit(self, ref: str) -> Commit:
         if not re.match(r'[0-9a-f]{40}', ref):
             if not ref.startswith(('heads/', 'refs/heads/')):
                 ref = 'refs/heads/' + ref
@@ -625,12 +628,11 @@ class Repo:
             ref = 'refs/' + ref
 
         r = self._session.get('https://api.github.com/repos/{}/commits/{}'.format(self.name, ref))
-        response = r.json()
-        assert 200 <= r.status_code < 300, response
+        assert 200 <= r.status_code < 300, r.json()
 
-        return self._commit_from_gh(response)
+        return self._commit_from_gh(r.json())
 
-    def _commit_from_gh(self, gh_commit):
+    def _commit_from_gh(self, gh_commit: dict) -> Commit:
         c = gh_commit['commit']
         return Commit(
             id=gh_commit['sha'],
@@ -788,7 +790,16 @@ class Repo:
         )).raise_for_status()
         return PR(self, number)
 
-    def make_pr(self, *, title=None, body=None, target, head, draft=False, token=None):
+    def make_pr(
+            self,
+            *,
+            title: Optional[str] = None,
+            body: Optional[str] = None,
+            target: str,
+            head: str,
+            draft: bool = False,
+            token: Optional[str] = None
+    ) -> PR:
         assert self.hook
         self.hook = 2
 
@@ -821,10 +832,9 @@ class Repo:
             },
             headers=headers,
         )
-        pr = r.json()
-        assert 200 <= r.status_code < 300, pr
+        assert 200 <= r.status_code < 300, r.json()
 
-        return PR(self, pr['number'])
+        return PR(self, r.json()['number'])
 
     def post_status(self, ref, status, context='default', **kw):
         assert self.hook

From 955a61a1e8dc16771702941d027b2bd340711a97 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 16 Oct 2023 10:46:29 +0200
Subject: [PATCH 077/192] [CHG] runbot_merge, forwardbot: merge commands parser

- move all commands parsing to runbot_merge as part of the long-term
  unification effort (#789)
- set up an actual parser-ish structure to parse the commands to
  something approaching a sum type (fixes #507)
- this is mostly prep for reworking the commands set (#673), although
  *strict command parsing* has been implemented (cf update to
  `test_unknown_commands`)
---
 forwardport/models/project.py               | 133 --------
 forwardport/tests/test_limit.py             |  22 +-
 forwardport/tests/test_weird.py             |   2 +-
 runbot_merge/models/commands.py             | 231 ++++++++++++++
 runbot_merge/models/project.py              |  54 +++-
 runbot_merge/models/pull_requests.py        | 322 +++++++++-----------
 runbot_merge/tests/test_basic.py            |  10 +-
 runbot_merge/tests/test_status_overrides.py |   2 +-
 8 files changed, 454 insertions(+), 322 deletions(-)
 create mode 100644 runbot_merge/models/commands.py

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 5aaf3a20..eb2b36b0 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -55,55 +55,6 @@ class Project(models.Model):
 
     id: int
     github_prefix: str
-    fp_github_token = fields.Char()
-    fp_github_name = fields.Char(store=True, compute="_compute_git_identity")
-    fp_github_email = fields.Char(store=True, compute="_compute_git_identity")
-
-    def _find_commands(self, comment):
-        if self.env.context.get('without_forward_port'):
-            return super()._find_commands(comment)
-
-        return re.findall(
-            '^\s*[@|#]?{}:? (.*)$'.format(self.fp_github_name),
-            comment, re.MULTILINE | re.IGNORECASE
-        ) + super()._find_commands(comment)
-
-    # technically the email could change at any moment...
-    @api.depends('fp_github_token')
-    def _compute_git_identity(self):
-        s = requests.Session()
-        for project in self:
-            if not project.fp_github_token or (project.fp_github_name and project.fp_github_email):
-                continue
-
-            r0 = s.get('https://api.github.com/user', headers={
-                'Authorization': 'token %s' % project.fp_github_token
-            })
-            if not r0.ok:
-                _logger.error("Failed to fetch forward bot information for project %s: %s", project.name, r0.text or r0.content)
-                continue
-
-            user = r0.json()
-            project.fp_github_name = user['name'] or user['login']
-            if email := user['email']:
-                project.fp_github_email = email
-                continue
-
-            if 'user:email' not in set(re.split(r',\s*', r0.headers['x-oauth-scopes'])):
-                raise UserError("The forward-port github token needs the user:email scope to fetch the bot's identity.")
-            r1 = s.get('https://api.github.com/user/emails', headers={
-                'Authorization': 'token %s' % project.fp_github_token
-            })
-            if not r1.ok:
-                _logger.error("Failed to fetch forward bot emails for project %s: %s", project.name, r1.text or r1.content)
-                continue
-            project.fp_github_email = next((
-                entry['email']
-                for entry in r1.json()
-                if entry['primary']
-            ), None)
-            if not project.fp_github_email:
-                raise UserError("The forward-port bot needs a public or primary email set up.")
 
     def write(self, vals):
         # check on branches both active and inactive so disabling branches doesn't
@@ -404,90 +355,6 @@ class PullRequests(models.Model):
             })
         return r
 
-    def _parse_commands(self, author, comment, login):
-        super(PullRequests, self.with_context(without_forward_port=True))._parse_commands(author, comment, login)
-
-        tokens = [
-            token
-            for line in re.findall('^\s*[@|#]?{}:? (.*)$'.format(self.repository.project_id.fp_github_name), comment['body'] or '', re.MULTILINE | re.IGNORECASE)
-            for token in line.split()
-        ]
-        if not tokens:
-            _logger.info("found no commands in comment of %s (%s) (%s)", author.github_login, author.display_name,
-                 utils.shorten(comment['body'] or '', 50)
-            )
-            return
-
-        # TODO: don't use a mutable tokens iterator
-        tokens = iter(tokens)
-        while True:
-            token = next(tokens, None)
-            if token is None:
-                break
-
-            ping = False
-            close = False
-            msg = None
-            if token in ('ci', 'skipci'):
-                pr = (self.source_id or self)
-                if pr._pr_acl(author).is_reviewer:
-                    pr.fw_policy = token
-                    msg = "Not waiting for CI to create followup forward-ports." if token == 'skipci' else "Waiting for CI to create followup forward-ports."
-                else:
-                    ping = True
-                    msg = "you can't configure ci."
-
-            if token == 'ignore': # replace 'ignore' by 'up to <pr_branch>'
-                token = 'up'
-                tokens = itertools.chain(['to', self.target.name], tokens)
-
-            if token in ('r+', 'review+'):
-                if not self.source_id:
-                    ping = True
-                    msg = "I can only do this on forward-port PRs and this is not one, see {}.".format(
-                        self.repository.project_id.github_prefix
-                    )
-                elif not self.parent_id:
-                    ping = True
-                    msg = "I can only do this on unmodified forward-port PRs, ask {}.".format(
-                        self.repository.project_id.github_prefix
-                    )
-                else:
-                    merge_bot = self.repository.project_id.github_prefix
-                    # don't update the root ever
-                    for pr in (p for p in self._iter_ancestors() if p.parent_id if p.state in RPLUS):
-                        # only the author is delegated explicitely on the
-                        pr._parse_commands(author, {**comment, 'body': merge_bot + ' r+'}, login)
-            elif token == 'close':
-                if self.source_id._pr_acl(author).is_reviewer:
-                    close = True
-                else:
-                    ping = True
-                    msg = "you can't close PRs."
-
-            elif token == 'up' and next(tokens, None) == 'to':
-                limit = next(tokens, None)
-                ping = True
-                if not self._pr_acl(author).is_author:
-                    msg = "you can't set a forward-port limit."
-                elif not limit:
-                    msg = "please provide a branch to forward-port to."
-                else:
-                    ping, msg = self._maybe_update_limit(limit)
-
-            if msg or close:
-                if msg:
-                    _logger.info("%s [%s]: %s", self.display_name, login, msg)
-                else:
-                    _logger.info("%s [%s]: closing", self.display_name, login)
-                self.env['runbot_merge.pull_requests.feedback'].create({
-                    'repository': self.repository.id,
-                    'pull_request': self.number,
-                    'message': f'@{author.github_login} {msg}' if msg and ping else msg,
-                    'close': close,
-                    'token_field': 'fp_github_token',
-                })
-
     def _maybe_update_limit(self, limit: str) -> typing.Tuple[bool, str]:
         limit_id = self.env['runbot_merge.branch'].with_context(active_test=False).search([
             ('project_id', '=', self.repository.project_id.id),
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index c96fa853..a2c174c5 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -101,22 +101,22 @@ def test_disable(env, config, make_repo, users):
         pr = prod.make_pr(target='a', head='branch2')
         prod.post_status(c, 'success', 'legal/cla')
         prod.post_status(c, 'success', 'ci/runbot')
-        pr.post_comment('hansen r+\n%s up to' % bot_name, config['role_reviewer']['token'])
-        pr.post_comment('%s up to b' % bot_name, config['role_reviewer']['token'])
-        pr.post_comment('%s up to foo' % bot_name, config['role_reviewer']['token'])
-        pr.post_comment('%s up to c' % bot_name, config['role_reviewer']['token'])
+        pr.post_comment(f'hansen r+\n{bot_name} up to', config['role_reviewer']['token'])
+        pr.post_comment(f'{bot_name} up to b', config['role_reviewer']['token'])
+        pr.post_comment(f'{bot_name} up to foo', config['role_reviewer']['token'])
+        pr.post_comment(f'{bot_name} up to c', config['role_reviewer']['token'])
     env.run_crons()
 
     # use a set because git webhooks delays might lead to mis-ordered
     # responses and we don't care that much
     assert set(pr.comments) == {
-        (users['reviewer'], "hansen r+\n%s up to" % bot_name),
-        (users['other'], "@%s please provide a branch to forward-port to." % users['reviewer']),
-        (users['reviewer'], "%s up to b" % bot_name),
-        (users['other'], "@%s branch 'b' is disabled, it can't be used as a forward port target." % users['reviewer']),
-        (users['reviewer'], "%s up to foo" % bot_name),
-        (users['other'], "@%s there is no branch 'foo', it can't be used as a forward port target." % users['reviewer']),
-        (users['reviewer'], "%s up to c" % bot_name),
+        (users['reviewer'], f"hansen r+\n{bot_name} up to"),
+        (users['other'], "@{reviewer} please provide a branch to forward-port to.".format_map(users)),
+        (users['reviewer'], f"{bot_name} up to b"),
+        (users['other'], "@{reviewer} branch 'b' is disabled, it can't be used as a forward port target.".format_map(users)),
+        (users['reviewer'], f"{bot_name} up to foo"),
+        (users['other'], "@{reviewer} there is no branch 'foo', it can't be used as a forward port target.".format_map(users)),
+        (users['reviewer'], f"{bot_name} up to c"),
         (users['other'], "Forward-porting to 'c'."),
         seen(env, pr, users),
     }
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index 32bcf0db..1c1fc4ab 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -809,7 +809,7 @@ def test_approve_draft(env, config, make_repo, users):
     assert pr.comments == [
         (users['reviewer'], 'hansen r+'),
         seen(env, pr, users),
-        (users['user'], f"I'm sorry, @{users['reviewer']}: draft PRs can not be approved."),
+        (users['user'], f"@{users['reviewer']} draft PRs can not be approved."),
     ]
 
     with prod:
diff --git a/runbot_merge/models/commands.py b/runbot_merge/models/commands.py
new file mode 100644
index 00000000..aa597bb8
--- /dev/null
+++ b/runbot_merge/models/commands.py
@@ -0,0 +1,231 @@
+import enum
+from collections import abc
+from dataclasses import dataclass, field
+from typing import Iterator, List, Optional, Union
+
+
+def tokenize(line: str) -> Iterator[str]:
+    cur = ''
+    for c in line:
+        if c == '-' and not cur:
+            yield '-'
+        elif c in ' \t+=,':
+            if cur:
+                yield cur
+            cur = ''
+            if not c.isspace():
+                yield c
+        else:
+            cur += c
+
+    if cur:
+        yield cur
+
+def normalize(it: Iterator[str]) -> Iterator[str]:
+    """Converts shorthand tokens to expanded version
+    """
+    for t in it:
+        match t:
+            case 'r':
+                yield 'review'
+            case 'r-':
+                yield 'review'
+                yield '-'
+            case 'p':
+                yield 'priority'
+            case _:
+                yield t
+
+
+@dataclass
+class Peekable(abc.Iterator[str]):
+    it: Iterator[str]
+    memo: Optional[str] = None
+
+    def __iter__(self) -> Iterator[str]:
+        return self
+
+    def __next__(self) -> str:
+        if self.memo is not None:
+            v, self.memo = self.memo, None
+            return v
+        return next(self.it)
+
+    def peek(self) -> Optional[str]:
+        if self.memo is None:
+            self.memo = next(self.it, None)
+        return self.memo
+
+
+def assert_next(it: Iterator[str], val: str):
+    if (actual := next(it, None)) != val:
+        raise CommandError(f"expected {val!r}, got {actual!r}")
+
+
+class CommandError(Exception):
+    pass
+
+
+class Approve:
+    def __str__(self):
+        return 'review+'
+
+
+class Reject:
+    def __str__(self):
+        return 'review-'
+
+
+class MergeMethod(enum.Enum):
+    SQUASH = 'squash'
+    REBASE_FF = 'rebase-ff'
+    REBASE_MERGE = 'rebase-merge'
+    MERGE = 'merge'
+
+    def __str__(self):
+        return self.value
+
+
+class Retry:
+    def __str__(self):
+        return 'retry'
+
+
+class Check:
+    def __str__(self):
+        return 'check'
+
+
+@dataclass
+class Override:
+    statuses: List[str] = field(default_factory=list)
+
+    def __str__(self):
+        return f"override={','.join(self.statuses)}"
+
+
+@dataclass
+class Delegate:
+    users: List[str] = field(default_factory=list)
+
+    def __str__(self):
+        if not self.users:
+            return 'delegate+'
+        return f"delegate={','.join(self.users)}"
+
+
+class Priority(enum.IntEnum):
+    NUKE = 0
+    PRIORITIZE = 1
+    DEFAULT = 2
+
+
+Command = Union[Approve, Reject, MergeMethod, Retry, Check, Override, Delegate, Priority]
+
+
+def parse_mergebot(line: str) -> Iterator[Command]:
+    it = Peekable(normalize(tokenize(line)))
+    for token in it:
+        match token:
+            case 'review':
+                match next(it):
+                    case '+':
+                        yield Approve()
+                    case '-':
+                        yield Reject()
+                    case t:
+                        raise CommandError(f"unknown review {t!r}")
+            case 'squash':
+                yield MergeMethod.SQUASH
+            case 'rebase-ff':
+                yield MergeMethod.REBASE_FF
+            case 'rebase-merge':
+                yield MergeMethod.REBASE_MERGE
+            case 'merge':
+                yield MergeMethod.MERGE
+            case 'retry':
+                yield Retry()
+            case 'check':
+                yield Check()
+            case 'override':
+                assert_next(it, '=')
+                ci = [next(it)]
+                while it.peek() == ',':
+                    next(it)
+                    ci.append(next(it))
+                yield Override(ci)
+            case 'delegate':
+                match next(it):
+                    case '+':
+                        yield Delegate()
+                    case '=':
+                        delegates = [next(it).lstrip('#@')]
+                        while it.peek() == ',':
+                            next(it)
+                            delegates.append(next(it).lstrip('#@'))
+                        yield Delegate(delegates)
+                    case d:
+                        raise CommandError(f"unknown delegation {d!r}")
+            case 'priority':
+                assert_next(it, '=')
+                yield Priority(int(next(it)))
+            case c:
+                raise CommandError(f"unknown command {c!r}")
+
+
+class FWApprove:
+    def __str__(self):
+        return 'review+'
+
+
+@dataclass
+class CI:
+    run: bool
+    def __str__(self):
+        return 'ci' if self.run else 'skipci'
+
+
+@dataclass
+class Limit:
+    branch: Optional[str]
+
+    def __str__(self):
+        if self.branch is None:
+            return 'ignore'
+        return f'up to {self.branch}'
+
+
+class Close:
+    def __str__(self):
+        return 'close'
+
+
+FWCommand = Union[FWApprove, CI, Limit, Close]
+
+
+def parse_forwardbot(line: str) -> Iterator[FWCommand]:
+    it = Peekable(normalize(tokenize(line)))
+    for token in it:
+        match token:
+            case 'review':
+                match next(it):
+                    case '+':
+                        yield FWApprove()
+                    case t:
+                        raise CommandError(f"unknown review {t!r}", True)
+            case 'ci':
+                yield CI(True)
+            case 'skipci':
+                yield CI(False)
+            case 'ignore':
+                yield Limit(None)
+            case 'up':
+                assert_next(it, 'to')
+                if limit := next(it, None):
+                    yield Limit(limit)
+                else:
+                    raise CommandError("please provide a branch to forward-port to.", True)
+            case 'close':
+                yield Close()
+            case c:
+                raise CommandError(f"unknown command {c!r}", True)
diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index b686137c..9e335267 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -1,5 +1,6 @@
 import logging
 import re
+from typing import List, Tuple
 
 import requests
 import sentry_sdk
@@ -38,6 +39,9 @@ class Project(models.Model):
         help="Prefix (~bot name) used when sending commands from PR "
              "comments e.g. [hanson retry] or [hanson r+ p=1]",
     )
+    fp_github_token = fields.Char()
+    fp_github_name = fields.Char(store=True, compute="_compute_git_identity")
+    fp_github_email = fields.Char(store=True, compute="_compute_git_identity")
 
     batch_limit = fields.Integer(
         default=8, group_operator=None, help="Maximum number of PRs staged together")
@@ -96,6 +100,43 @@ class Project(models.Model):
             if not project.github_email:
                 raise UserError("The merge bot needs a public or accessible primary email set up.")
 
+    # technically the email could change at any moment...
+    @api.depends('fp_github_token')
+    def _compute_git_identity(self):
+        s = requests.Session()
+        for project in self:
+            if not project.fp_github_token or (project.fp_github_name and project.fp_github_email):
+                continue
+
+            r0 = s.get('https://api.github.com/user', headers={
+                'Authorization': 'token %s' % project.fp_github_token
+            })
+            if not r0.ok:
+                _logger.error("Failed to fetch forward bot information for project %s: %s", project.name, r0.text or r0.content)
+                continue
+
+            user = r0.json()
+            project.fp_github_name = user['name'] or user['login']
+            if email := user['email']:
+                project.fp_github_email = email
+                continue
+
+            if 'user:email' not in set(re.split(r',\s*', r0.headers['x-oauth-scopes'])):
+                raise UserError("The forward-port github token needs the user:email scope to fetch the bot's identity.")
+            r1 = s.get('https://api.github.com/user/emails', headers={
+                'Authorization': 'token %s' % project.fp_github_token
+            })
+            if not r1.ok:
+                _logger.error("Failed to fetch forward bot emails for project %s: %s", project.name, r1.text or r1.content)
+                continue
+            project.fp_github_email = next((
+                entry['email']
+                for entry in r1.json()
+                if entry['primary']
+            ), None)
+            if not project.fp_github_email:
+                raise UserError("The forward-port bot needs a public or primary email set up.")
+
     def _check_stagings(self, commit=False):
         # check branches with an active staging
         for branch in self.env['runbot_merge.branch']\
@@ -132,9 +173,18 @@ class Project(models.Model):
                 if commit:
                     self.env.cr.commit()
 
-    def _find_commands(self, comment):
+    def _find_commands(self, comment: str) -> List[Tuple[str, str]]:
+        """Tries to find all the lines starting (ignoring leading whitespace)
+        with either the merge or the forward port bot identifiers.
+
+        For convenience, the identifier *can* be prefixed with an ``@`` or
+        ``#``, and suffixed with a ``:``.
+        """
+        logins = '|'.join(map(re.escape, filter(None, [self.github_prefix, self.fp_github_name])))
+        # horizontal whitespace (\s - {\n, \r}), but Python doesn't have \h or \p{Blank}
+        h = r'[^\S\r\n]'
         return re.findall(
-            '^\s*[@|#]?{}:? (.*)$'.format(self.github_prefix),
+            fr'^{h}*[@|#]?({logins})(?:{h}+|:{h}*)(.*)$',
             comment, re.MULTILINE | re.IGNORECASE)
 
     def _has_branch(self, name):
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 1b7a120b..6e4d63db 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -8,7 +8,7 @@ import logging
 import pprint
 import re
 import time
-from typing import Optional, Union
+from typing import Optional, Union, List, Literal
 
 import sentry_sdk
 import werkzeug
@@ -16,6 +16,7 @@ import werkzeug
 from odoo import api, fields, models, tools
 from odoo.exceptions import ValidationError
 from odoo.osv import expression
+from . import commands
 
 from .. import github, exceptions, controllers, utils
 
@@ -561,71 +562,46 @@ class PullRequests(models.Model):
             'closing': closing,
         })
 
-    def _parse_command(self, commandstring):
-        for m in re.finditer(
-            r'(\S+?)(?:([+-])|=(\S*))?(?=\s|$)',
-            commandstring,
-        ):
-            name, flag, param = m.groups()
-            if name == 'r':
-                name = 'review'
-            if flag in ('+', '-'):
-                yield name, flag == '+'
-            elif name == 'delegate':
-                if param:
-                    for p in param.split(','):
-                        yield 'delegate', p.lstrip('#@')
-            elif name == 'override':
-                if param:
-                    for p in param.split(','):
-                        yield 'override', p
-            elif name in ('p', 'priority'):
-                if param in ('0', '1', '2'):
-                    yield ('priority', int(param))
-            elif any(name == k for k, _ in type(self).merge_method.selection):
-                yield ('method', name)
-            else:
-                yield name, param
-
     def _parse_commands(self, author, comment, login):
-        """Parses a command string prefixed by Project::github_prefix.
-
-        A command string can contain any number of space-separated commands:
-
-        retry
-          resets a PR in error mode to ready for staging
-        r(eview)+/-
-           approves or disapproves a PR (disapproving just cancels an approval)
-        delegate+/delegate=<users>
-          adds either PR author or the specified (github) users as
-          authorised reviewers for this PR. ``<users>`` is a
-          comma-separated list of github usernames (no @)
-        p(riority)=2|1|0
-          sets the priority to normal (2), pressing (1) or urgent (0).
-          Lower-priority PRs are selected first and batched together.
-        rebase+/-
-          Whether the PR should be rebased-and-merged (the default) or just
-          merged normally.
-        """
         assert self, "parsing commands must be executed in an actual PR"
 
         (login, name) = (author.github_login, author.display_name) if author else (login, 'not in system')
 
-        is_admin, is_reviewer, is_author = self._pr_acl(author)
-
-        commands = [
-            ps
-            for m in self.repository.project_id._find_commands(comment['body'] or '')
-            for ps in self._parse_command(m)
-        ]
-
-        if not commands:
-            _logger.info("found no commands in comment of %s (%s) (%s)", author.github_login, author.display_name,
+        commandlines = self.repository.project_id._find_commands(comment['body'] or '')
+        if not commandlines:
+            _logger.info("found no commands in comment of %s (%s) (%s)", login, name,
                  utils.shorten(comment['body'] or '', 50)
             )
             return 'ok'
 
-        if not (is_author or any(cmd == 'override' for cmd, _ in commands)):
+        def feedback(message: Optional[str] = None, close: bool = False, token: Literal['github_token', 'fp_github_token'] = 'github_token'):
+            self.env['runbot_merge.pull_requests.feedback'].create({
+                'repository': self.repository.id,
+                'pull_request': self.number,
+                'message': message,
+                'close': close,
+                'token_field': token,
+            })
+        try:
+            cmds: List[Union[commands.Command, commands.FWCommand]] = [
+                ps
+                for bot, line in commandlines
+                for ps in (commands.parse_mergebot(line) if bot.casefold() == self.repository.project_id.github_prefix.casefold() else commands.parse_forwardbot(line))
+            ]
+        except Exception as e:
+            _logger.info(
+                "error %s while parsing comment of %s (%s): %s",
+                e,
+                author.github_login, author.display_name,
+                utils.shorten(comment['body'] or '', 50),
+                exc_info=True
+            )
+            feedback(message=f"@{login} {e.args[0]}", token='fp_github_token' if len(e.args) >= 2 and e.args[1] else 'github_token')
+            return 'error'
+
+        is_admin, is_reviewer, is_author = self._pr_acl(author)
+
+        if not (is_author or any(isinstance(cmd, commands.Override) for cmd in cmds)):
             # no point even parsing commands
             _logger.info("ignoring comment of %s (%s): no ACL to %s",
                           login, name, self.display_name)
@@ -636,40 +612,13 @@ class PullRequests(models.Model):
             )
             return 'ignored'
 
-        applied, ignored = [], []
-        def reformat(command, param):
-            if param is None:
-                pstr = ''
-            elif isinstance(param, bool):
-                pstr = '+' if param else '-'
-            elif isinstance(param, list):
-                pstr = '=' + ','.join(param)
-            else:
-                pstr = '={}'.format(param)
-
-            return '%s%s' % (command, pstr)
-        msgs = []
-        for command, param in commands:
-            ok = False
-            msg = None
-            if command == 'retry':
-                if is_author:
-                    if self.state == 'error':
-                        ok = True
-                        self.state = 'ready'
-                    else:
-                        msg = "retry makes no sense when the PR is not in error."
-            elif command == 'check':
-                if is_author:
-                    self.env['runbot_merge.fetch_job'].create({
-                        'repository': self.repository.id,
-                        'number': self.number,
-                    })
-                    ok = True
-            elif command == 'review':
-                if self.draft:
+        rejections = []
+        for command in cmds:
+            fwbot, msg = False, None
+            match command:
+                case commands.Approve() if self.draft:
                     msg = "draft PRs can not be approved."
-                elif param and is_reviewer:
+                case commands.Approve() if is_reviewer:
                     oldstate = self.state
                     newstate = RPLUS.get(self.state)
                     if not author.email:
@@ -679,7 +628,6 @@ class PullRequests(models.Model):
                     else:
                         self.state = newstate
                         self.reviewed_by = author
-                        ok = True
                     _logger.debug(
                         "r+ on %s by %s (%s->%s) status=%s message? %s",
                         self.display_name, author.github_login,
@@ -694,7 +642,7 @@ class PullRequests(models.Model):
                             pull_request=self.number,
                             format_args={'user': login, 'pr': self},
                         )
-                elif not param and is_author:
+                case commands.Reject() if is_author:
                     newstate = RMINUS.get(self.state)
                     if self.priority == 0 or newstate:
                         if newstate:
@@ -707,94 +655,128 @@ class PullRequests(models.Model):
                                 format_args={'user': login, 'pr': self},
                             )
                         self.unstage("unreviewed (r-) by %s", login)
-                        ok = True
                     else:
                         msg = "r- makes no sense in the current PR state."
-            elif command == 'delegate':
-                if is_reviewer:
-                    ok = True
-                    Partners = self.env['res.partner']
-                    if param is True:
-                        delegate = self.author
-                    else:
-                        delegate = Partners.search([('github_login', '=', param)]) or Partners.create({
-                            'name': param,
-                            'github_login': param,
-                        })
-                    delegate.write({'delegate_reviewer': [(4, self.id, 0)]})
-            elif command == 'priority':
-                if is_admin:
-                    ok = True
-                    self.priority = param
-                    if param == 0:
-                        self.target.active_staging_id.cancel(
-                            "P=0 on %s by %s, unstaging target %s",
-                            self.display_name,
-                            author.github_login, self.target.name,
-                        )
-            elif command == 'method':
-                if is_reviewer:
-                    self.merge_method = param
-                    ok = True
-                    explanation = next(label for value, label in type(self).merge_method.selection if value == param)
+                case commands.MergeMethod() as command if is_reviewer:
+                    self.merge_method = command.value
+                    explanation = next(label for value, label in type(self).merge_method.selection if value == command.value)
                     self.env.ref("runbot_merge.command.method")._send(
                         repository=self.repository,
                         pull_request=self.number,
                         format_args={'new_method': explanation, 'pr': self, 'user': login},
                     )
-            elif command == 'override':
-                overridable = author.override_rights\
-                    .filtered(lambda r: not r.repository_id or (r.repository_id == self.repository))\
-                    .mapped('context')
-                if param in overridable:
-                    self.overrides = json.dumps({
-                        **json.loads(self.overrides),
-                        param: {
-                            'state': 'success',
-                            'target_url': comment['html_url'],
-                            'description': f"Overridden by @{author.github_login}",
-                        },
-                    })
-                    c = self.env['runbot_merge.commit'].search([('sha', '=', self.head)])
-                    if c:
-                        c.to_check = True
+                case commands.Retry() if is_author:
+                    if self.state == 'error':
+                        self.state = 'ready'
                     else:
-                        c.create({'sha': self.head, 'statuses': '{}'})
-                    ok = True
-                else:
-                    msg = "you are not allowed to override this status."
-            else:
-                # ignore unknown commands
-                continue
+                        msg = "retry makes no sense when the PR is not in error."
+                case commands.Check() if is_author:
+                    self.env['runbot_merge.fetch_job'].create({
+                        'repository': self.repository.id,
+                        'number': self.number,
+                    })
+                case commands.Delegate(users) if is_reviewer:
+                    if not users:
+                        delegates = self.author
+                    else:
+                        delegates = self.env['res.partner']
+                        for login in users:
+                            delegates |= delegates.search([('github_login', '=', login)]) or delegates.create({
+                                'name': login,
+                                'github_login': login,
+                            })
+                    delegates.write({'delegate_reviewer': [(4, self.id, 0)]})
+                case commands.Priority() if is_admin:
+                    self.priority = int(command)
+                    if command is commands.Priority.NUKE:
+                        self.target.active_staging_id.cancel(
+                            "P=0 on %s by %s, unstaging target %s",
+                            self.display_name,
+                            author.github_login, self.target.name,
+                        )
+                case commands.Override(statuses):
+                    for status in statuses:
+                        overridable = author.override_rights\
+                            .filtered(lambda r: not r.repository_id or (r.repository_id == self.repository))\
+                            .mapped('context')
+                        if status in overridable:
+                            self.overrides = json.dumps({
+                                **json.loads(self.overrides),
+                                status: {
+                                    'state': 'success',
+                                    'target_url': comment['html_url'],
+                                    'description': f"Overridden by @{author.github_login}",
+                                },
+                            })
+                            c = self.env['runbot_merge.commit'].search([('sha', '=', self.head)])
+                            if c:
+                                c.to_check = True
+                            else:
+                                c.create({'sha': self.head, 'statuses': '{}'})
+                        else:
+                            msg = f"you are not allowed to override {status!r}."
+                # FW
+                case commands.FWApprove():
+                    fwbot = True
+                    if not self.source_id:
+                        msg = "I can only do this on forward-port PRs and this is not one, see {}.".format(
+                            self.repository.project_id.github_prefix
+                        )
+                    elif not self.parent_id:
+                        msg = "I can only do this on unmodified forward-port PRs, ask {}.".format(
+                            self.repository.project_id.github_prefix
+                        )
+                    else:
+                        merge_bot = self.repository.project_id.github_prefix
+                        # FIXME: classification of messages from child PR :(
+                        # don't update the root ever
+                        for pr in (p for p in self._iter_ancestors() if p.parent_id if p.state in RPLUS):
+                            # only the author is delegated explicitely on the
+                            pr._parse_commands(author, {**comment, 'body': merge_bot + ' r+'}, login)
+                case commands.Close() if self.source_id._pr_acl(author).is_reviewer:
+                    feedback(close=True, token='fp_github_token')
+                case commands.CI(run):
+                    pr = (self.source_id or self)
+                    if pr._pr_acl(author).is_reviewer:
+                        pr.fw_policy = 'ci' if run else 'skipci'
+                        feedback(
+                            message="Waiting for CI to create followup forward-ports." if run else "Not waiting for CI to create followup forward-ports.",
+                            token='fp_github_token',
+                        )
+                    else:
+                        fwbot = True
+                        msg = "you can't configure ci."
+                case commands.Limit(branch):
+                    fwbot = True
+                    if is_author:
+                        ping, msg = self._maybe_update_limit(branch or self.target.name)
+                        if not ping:
+                            feedback(message=msg, token='fp_github_token')
+                            msg = None
+                    else:
+                        msg = "you can't set a forward-port limit."
+                # NO!
+                case _:
+                    msg = f"you can't {command}. Skill issue."
+            if msg is not None:
+                rejections.append((fwbot, msg))
 
-            _logger.info(
-                "%s %s(%s) on %s by %s (%s)",
-                "applied" if ok else "ignored",
-                command, param, self.display_name,
-                author.github_login, author.display_name,
-            )
-            if ok:
-                applied.append(reformat(command, param))
-            else:
-                ignored.append(reformat(command, param))
-                msgs.append(msg or "you can't {}.".format(reformat(command, param)))
+        if not rejections:
+            _logger.info("%s (%s) applied %s", login, name, cmds)
+            return 'applied ' + ', '.join(map(str, cmds))
 
-        if msgs:
-            joiner = ' ' if len(msgs) == 1 else '\n- '
-            msgs.insert(0, "I'm sorry, @{}:".format(login))
-            self.env['runbot_merge.pull_requests.feedback'].create({
-                'repository': self.repository.id,
-                'pull_request': self.number,
-                'message': joiner.join(msgs),
-            })
-
-        msg = []
-        if applied:
-            msg.append('applied ' + ' '.join(applied))
-        if ignored:
-            ignoredstr = ' '.join(ignored)
-            msg.append('ignored ' + ignoredstr)
-        return '\n'.join(msg)
+        self.env.cr.rollback()
+        rejections_list = ''.join(f'\n- {r}' for fw, r in rejections if not fw)
+        fw_rejections_list = ''.join(f'\n- {r}' for fw, r in rejections if fw)
+        _logger.info("%s (%s) tried to apply %s%s", login, name, cmds, rejections_list + fw_rejections_list)
+        footer = '' if len(cmds) == len(rejections) else "\n\nFor your own safety I've ignored everything in your comment."
+        if rejections_list:
+            rejections = ' ' + rejections_list.removeprefix("\n- ") if rejections_list.count('\n- ') == 1 else rejections_list
+            feedback(message=f"@{login}{rejections}{footer}")
+        if fw_rejections_list:
+            rejections = ' ' + fw_rejections_list.removeprefix("\n- ") if fw_rejections_list.count('\n- ') else fw_rejections_list
+            feedback(message=f"@{login}{rejections}{footer}", token='fp_github_token')
+        return 'rejected'
 
     def _pr_acl(self, user):
         if not self:
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 1244bda2..5ddff161 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -1222,7 +1222,7 @@ class TestRetry:
             (users['reviewer'], 'hansen r+'),
             (users['reviewer'], 'hansen retry'),
             seen(env, prx, users),
-            (users['user'], "I'm sorry, @{reviewer}: retry makes no sense when the PR is not in error.".format_map(users)),
+            (users['user'], "@{reviewer} retry makes no sense when the PR is not in error.".format_map(users)),
         ]
 
     @pytest.mark.parametrize('disabler', ['user', 'other', 'reviewer'])
@@ -2909,7 +2909,7 @@ class TestReviewing(object):
             (users['user'], "I'm sorry, @{}. I'm afraid I can't do that.".format(users['other'])),
             (users['reviewer'], 'hansen r+'),
             (users['reviewer'], 'hansen r+'),
-            (users['user'], "I'm sorry, @{}: this PR is already reviewed, reviewing it again is useless.".format(
+            (users['user'], "@{} this PR is already reviewed, reviewing it again is useless.".format(
                  users['reviewer'])),
         ]
 
@@ -2937,7 +2937,7 @@ class TestReviewing(object):
         assert prx.comments == [
             (users['reviewer'], 'hansen r+'),
             seen(env, prx, users),
-            (users['user'], "I'm sorry, @{}: you can't review+.".format(users['reviewer'])),
+            (users['user'], "@{} you can't review+. Skill issue.".format(users['reviewer'])),
         ]
 
     def test_self_review_success(self, env, repo, users, config):
@@ -3092,7 +3092,7 @@ class TestReviewing(object):
             seen(env, pr, users),
             (users['reviewer'], 'hansen delegate+'),
             (users['user'], 'hansen r+'),
-            (users['user'], f"I'm sorry, @{users['user']}: I must know your email before you can review PRs. Please contact an administrator."),
+            (users['user'], f"@{users['user']} I must know your email before you can review PRs. Please contact an administrator."),
         ]
         user_partner.fetch_github_email()
         assert user_partner.email
@@ -3394,6 +3394,8 @@ class TestRecognizeCommands:
             (users['reviewer'], "hansen do the thing"),
             (users['reviewer'], "hansen @bobby-b r+ :+1:"),
             seen(env, pr, users),
+            (users['user'], "@{reviewer} unknown command 'do'".format_map(users)),
+            (users['user'], "@{reviewer} unknown command '@bobby-b'".format_map(users)),
         ]
 
 class TestRMinus:
diff --git a/runbot_merge/tests/test_status_overrides.py b/runbot_merge/tests/test_status_overrides.py
index 9ffe7604..1bdc4877 100644
--- a/runbot_merge/tests/test_status_overrides.py
+++ b/runbot_merge/tests/test_status_overrides.py
@@ -89,7 +89,7 @@ def test_basic(env, project, make_repo, users, setreviewers, config):
         (users['reviewer'], 'hansen r+'),
         seen(env, pr, users),
         (users['reviewer'], 'hansen override=l/int'),
-        (users['user'], "I'm sorry, @{}: you are not allowed to override this status.".format(users['reviewer'])),
+        (users['user'], "@{} you are not allowed to override 'l/int'.".format(users['reviewer'])),
         (users['other'], "hansen override=l/int"),
     ]
     assert pr_id.statuses == '{}'

From 75f29f9315a04a540222aba4f2440dca13607080 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 8 Apr 2024 13:23:56 +0200
Subject: [PATCH 078/192] [IMP] conftest: avoid parsing json it if may not be

When asserting a status fails, it's possible that this is a 400 or
500-type error which does not yield JSON data at all (e.g. forgot to
start the proxy or dummy), in which case trying to dump the json data
is actively harmful as it triggers cascading failures. And it's not
like the output message is formatted, so a re-structured assertion
message is not helpful.
---
 conftest.py | 43 +++++++++++++++++++++----------------------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/conftest.py b/conftest.py
index 090ebb4b..2bf6f4e0 100644
--- a/conftest.py
+++ b/conftest.py
@@ -585,14 +585,13 @@ class Repo:
         assert self.hook
         r = self._session.get(
             'https://api.github.com/repos/{}/hooks'.format(self.name))
-        response = r.json()
-        assert 200 <= r.status_code < 300, response
-        [hook] = response
+        assert 200 <= r.status_code < 300, r.text
+        [hook] = r.json()
 
         r = self._session.patch('https://api.github.com/repos/{}/hooks/{}'.format(self.name, hook['id']), json={
             'config': {**hook['config'], 'secret': secret},
         })
-        assert 200 <= r.status_code < 300, r.json()
+        assert 200 <= r.status_code < 300, r.text
 
     def get_ref(self, ref):
         # differs from .commit(ref).id for the sake of assertion error messages
@@ -628,7 +627,7 @@ class Repo:
             ref = 'refs/' + ref
 
         r = self._session.get('https://api.github.com/repos/{}/commits/{}'.format(self.name, ref))
-        assert 200 <= r.status_code < 300, r.json()
+        assert 200 <= r.status_code < 300, r.text
 
         return self._commit_from_gh(r.json())
 
@@ -650,14 +649,14 @@ class Repo:
         :rtype: Dict[str, str]
         """
         r = self._session.get('https://api.github.com/repos/{}/git/trees/{}'.format(self.name, commit.tree))
-        assert 200 <= r.status_code < 300, r.json()
+        assert 200 <= r.status_code < 300, r.text
 
         # read tree's blobs
         tree = {}
         for t in r.json()['tree']:
             assert t['type'] == 'blob', "we're *not* doing recursive trees in test cases"
             r = self._session.get('https://api.github.com/repos/{}/git/blobs/{}'.format(self.name, t['sha']))
-            assert 200 <= r.status_code < 300, r.json()
+            assert 200 <= r.status_code < 300, r.text
             tree[t['path']] = base64.b64decode(r.json()['content']).decode()
 
         return tree
@@ -687,7 +686,7 @@ class Repo:
             'required_pull_request_reviews': None,
             'restrictions': None,
         })
-        assert 200 <= r.status_code < 300, r.json()
+        assert 200 <= r.status_code < 300, r.text
 
     # FIXME: remove this (runbot_merge should use make_commits directly)
     def make_commit(self, ref, message, author, committer=None, tree=None, wait=True):
@@ -832,7 +831,7 @@ class Repo:
             },
             headers=headers,
         )
-        assert 200 <= r.status_code < 300, r.json()
+        assert 200 <= r.status_code < 300, r.text
 
         return PR(self, r.json()['number'])
 
@@ -845,7 +844,7 @@ class Repo:
             'context': context,
             **kw
         })
-        assert 200 <= r.status_code < 300, r.json()
+        assert 200 <= r.status_code < 300, r.text
 
     def is_ancestor(self, sha, of):
         return any(c['sha'] == sha for c in self.log(of))
@@ -856,7 +855,7 @@ class Repo:
                 'https://api.github.com/repos/{}/commits'.format(self.name),
                 params={'sha': ref_or_sha, 'page': page}
             )
-            assert 200 <= r.status_code < 300, r.json()
+            assert 200 <= r.status_code < 300, r.text
             yield from r.json()
             if not r.links.get('next'):
                 return
@@ -924,7 +923,7 @@ class PR:
             'https://api.github.com/repos/{}/pulls/{}'.format(self.repo.name, self.number),
             headers=caching
         )
-        assert r.ok, r.json()
+        assert r.ok, r.text
         if r.status_code == 304:
             return previous
         contents, caching = self._cache = r.json(), {}
@@ -969,7 +968,7 @@ class PR:
     @property
     def comments(self):
         r = self.repo._session.get('https://api.github.com/repos/{}/issues/{}/comments'.format(self.repo.name, self.number))
-        assert 200 <= r.status_code < 300, r.json()
+        assert 200 <= r.status_code < 300, r.text
         return [Comment(c) for c in r.json()]
 
     @property
@@ -986,7 +985,7 @@ class PR:
             json={'body': body},
             headers=headers,
         )
-        assert 200 <= r.status_code < 300, r.json()
+        assert 200 <= r.status_code < 300, r.text
         return r.json()['id']
 
     def edit_comment(self, cid, body, token=None):
@@ -999,7 +998,7 @@ class PR:
             json={'body': body},
             headers=headers
         )
-        assert 200 <= r.status_code < 300, r.json()
+        assert 200 <= r.status_code < 300, r.text
         wait_for_hook()
 
     def delete_comment(self, cid, token=None):
@@ -1011,7 +1010,7 @@ class PR:
             'https://api.github.com/repos/{}/issues/comments/{}'.format(self.repo.name, cid),
             headers=headers
         )
-        assert r.status_code == 204, r.json()
+        assert r.status_code == 204, r.text
 
     def _set_prop(self, prop, value, token=None):
         assert self.repo.hook
@@ -1035,7 +1034,7 @@ class PR:
             self.repo.name,
             self.number,
         ))
-        assert 200 <= r.status_code < 300, r.json()
+        assert 200 <= r.status_code < 300, r.text
         info = r.json()
 
         repo = self.repo
@@ -1056,7 +1055,7 @@ class PR:
             json={'body': body, 'event': state,},
             headers=headers
         )
-        assert 200 <= r.status_code < 300, r.json()
+        assert 200 <= r.status_code < 300, r.text
 
 PRBranch = collections.namedtuple('PRBranch', 'repo branch')
 class LabelsProxy(collections.abc.MutableSet):
@@ -1067,7 +1066,7 @@ class LabelsProxy(collections.abc.MutableSet):
     def _labels(self):
         pr = self._pr
         r = pr.repo._session.get('https://api.github.com/repos/{}/issues/{}/labels'.format(pr.repo.name, pr.number))
-        assert r.ok, r.json()
+        assert r.ok, r.text
         return {label['name'] for label in r.json()}
 
     def __repr__(self):
@@ -1093,14 +1092,14 @@ class LabelsProxy(collections.abc.MutableSet):
         r = pr.repo._session.post('https://api.github.com/repos/{}/issues/{}/labels'.format(pr.repo.name, pr.number), json={
             'labels': [label]
         })
-        assert r.ok, r.json()
+        assert r.ok, r.text
 
     def discard(self, label):
         pr = self._pr
         assert pr.repo.hook
         r = pr.repo._session.delete('https://api.github.com/repos/{}/issues/{}/labels/{}'.format(pr.repo.name, pr.number, label))
         # discard should do nothing if the item didn't exist in the set
-        assert r.ok or r.status_code == 404, r.json()
+        assert r.ok or r.status_code == 404, r.text
 
     def update(self, *others):
         pr = self._pr
@@ -1109,7 +1108,7 @@ class LabelsProxy(collections.abc.MutableSet):
         r = pr.repo._session.post('https://api.github.com/repos/{}/issues/{}/labels'.format(pr.repo.name, pr.number), json={
             'labels': list(set(itertools.chain.from_iterable(others)))
         })
-        assert r.ok, r.json()
+        assert r.ok, r.text
 
 class Environment:
     def __init__(self, port, db, default_crons=()):

From d4fa1fd35315d330566e37f515a937f722859ef7 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 31 Oct 2023 07:42:07 +0100
Subject: [PATCH 079/192] [CHG] *: rewrite commands set, rework status
 management

This commit revisits the commands set in order to make it more
regular, and limit inconsistent command-sets, although it includes
pseudo-command aliases for common tasks now removed from the core set.

Hard Errors
===========

The previous iteration of the commands set would ignore any
non-command term in a command line. This has been changed to hard
error (and ignoring the entire thing) if any command is unknown or
invalid.

This fixes inconsistent / unexpected interpretations where a user
sends a command, then writes a novel on the same line some words of
which happen to *also* be commands, leading to merge states they did
not expect. They should now be told to fuck off.

Priority Restructuring
----------------------

The numerical priority system was pretty messy in that it confused
"staging priority" (in ways which were not entirely straightforward)
with overrides to other concerns.

This has now being split along all the axis, with separate command
subsets for:

- staging prioritisation, now separated between `default`, `priority`,
  and `alone`,

  - `default` means PRs are picked by an unspecified order when
    creating a staging, if nothing better is available
  - `priority` means PRs are picked first when staging, however if
    `priority` PRs don't fill the staging the rest will be filled with
    `default`, this mode did not previously exist
  - `alone` means the PRs are picked first, before splits, and only
    `alone` PRs can be part of the staging (which usually matches the
    modename)
- `skipchecks` overrides both statuses and approval checks, for the
  batch, something previously implied in `p=0`, but now
  independent. Setting `skipchecks` basically makes the entire batch
  `ready`.

  For consistency this also sets the reviewer implicitly: since
  skipchecks overrides both statuses *and approval*, whoever enables
  this mode is essentially the reviewer.
- `cancel` cancels any ongoing staging when the marked PR becomes
  ready again, previously this was also implied (in a more restricted
  form) by setting `p=0`

FWBot removal
=============

While the "forwardport bot" still exists as an API level (to segregate
access rights between tokens) it has been removed as an interaction
point, as part of the modules merge plan. As a result,

fwbot stops responding
----------------------

Feedback messages are now always sent by the mergebot, the
forward-porting bot should not send any message or notification
anymore.

commands moved to the merge bot
-------------------------------

- `ignore`/`up to` simply changes bot
- `close` as well
- `skipci` is now a choice / flag of an `fw` command, which denotes
  the forward-port policy,

  - `fw=default` is the old `ci` and resets the policy to default,
    that is wait for the PR to be merged to create forward ports, and
    for the required statuses on each forward port to be received
    before creating the next
  - `fw=skipci` is the old `skipci`, it waits for the merge of the
    base PR but then creates all the forward ports immediately (unless
    it gets a conflict)
  - `fw=skipmerge` immediately creates all the forward ports, without
    even waiting for the PR to be merged

    This is a completely new mode, and may be rather broken as until
    now the 'bot has always assumed the source PR had been merged.

approval rework
---------------

Because of the previous section, there is no distinguishing feature
between `mergebot r+` = "merge this PR" and `forwardbot r+` = "merge
this PR and all its parent with different access rights".

As a result, the two have been merged under a single `mergebot r+`
with heuristics attempting to provide the best experience:

- if approving a non-forward port, the behavior does not change
- else, with review rights on the source, all ancestors are approved
- else, as author of the original, approves all ancestors which descend
  from a merged PR
- else, approves all ancestors up to and including the oldest ancestor
  to which we have review rights

Most notably, the source's author is not delegated on the source or
any of its descendants anymore. This might need to be revisited if it
provides too restrictive.

For the very specialized need of approving a forward-port *and none of
its ancestors*, `review=` can now take a comma (`,`) separated list of
pull request numbers (github numbers, not mergebot ids).

Computed State
==============

The `state` field of pull requests is now computed. Hopefully this
makes the status more consistent and predictable in the long run, and
importantly makes status management more reliable (because reference
datum get updated naturally flowing to the state).

For now however it makes things more complicated as some of the states
have to be separately signaled or updated:

- `closed` and `error` are now separate flags
- `merge_date` is pulled down from forwardport and becomes the
  transition signal for ready -> merged
- `reviewed_by` becomes the transition signal for approval (might be a
  good idea to rename it...)
- `status` is computed from the head's statuses and overrides, and
  *that* becomes the validation state

Ideally, batch-level flags like `skipchecks` should be on, well, the
batch, and `state` should have a dependency on the batch. However
currently the batch is not a durable / permanent member of the system,
so it's a PR-level flag and a messy pile.

On notable change is that *forcing* the state to `ready` now does that
but also sets the reviewer, `skipchecks`, and overrides to ensure the
API-mediated readying does not get rolled back by e.g. the runbot
sending a status.

This is useful for a few types of automated / programmatic PRs
e.g. translation exports, where we set the state programmatically to
limit noise.

recursive dependency hack
-------------------------

Given a sequence of PRs with an override of the source, if one of the
PRs is updated its descendants should not have the override
anymore. However if the updated PR gets overridden, its descendants
should have *that* override.

This requires some unholy manipulations via an override of `modified`,
as the ORM supports recursive fields but not recursive
dependencies (on a different field).

unconditional followup scheduling
---------------------------------

Previously scheduling forward-port followup was contigent on the FW
policy, but it's not actually correct if the new PR is *immediately*
validated (which can happen now that the field is computed, if there
are no required statuses *or* all of the required statuses are
overridden by an ancestor) as nothing will trigger the state change
and thus scheduling of the fp followup.

The followup function checks all the properties of the batch to port,
so this should not result on incorrect ports. Although it's a bit more
expensive, and will lead to more spam.

Previously this would not happen because on creation of a PR the
validation task (commit -> PR) would still have to execute.

Misc Changes
============

- If a PR is marked as overriding / canceling stagings, it now does
  so on retry not just when setting initially.

  This was not handled at all previously, so a PR in P0 going into
  error due to e.g. a non-deterministic bug would be retried and still
  p=0, but a current staging would not get cancelled. Same when a PR
  in p=0 goes into error because something was failed, then is updated
  with a fix.
- Add tracking to a bunch of relevant PR fields.

  Post-mortem analysis currently generally requires going through the
  text logs to see what happened, which is annoying.

  There is a nondeterminism / inconsistency in the tracking which
  sometimes leads the admin user to trigger tracking before the bot
  does, leading to the staging tracking being attributed to them
  during tests, shove under the carpet by ignoring the user to whom
  that tracking is attributed.

  When multiple users update tracked fields in the same transaction
  all the changes are attributed to the first one having triggered
  tracking (?), I couldn't find why the admin sometimes takes over.
- added and leveraged support for enum-backed selection fields
- moved variuous fields from forwardport to runbot_merge
- fix a migration which had never worked and which never run (because
  I forgot to bump the version on the module)
- remove some unnecessary intermediate de/serialisation

fixes #673, fixes #309, fixes #792, fixes #846 (probably)
---
 forwardport/__manifest__.py                   |   2 +-
 forwardport/data/views.xml                    |  34 -
 .../migrations/15.0.1.4/pre-migration.py      |   7 +
 forwardport/models/project.py                 |  99 +--
 forwardport/tests/test_conflicts.py           |   7 +-
 forwardport/tests/test_limit.py               |  73 +-
 forwardport/tests/test_overrides.py           |  38 +-
 forwardport/tests/test_simple.py              |  59 +-
 forwardport/tests/test_updates.py             |  24 +-
 forwardport/tests/test_weird.py               |  20 +-
 mergebot_test_utils/utils.py                  |   5 +-
 runbot_merge/__manifest__.py                  |   4 +-
 runbot_merge/changelog/2023-12/commands.md    |  57 ++
 runbot_merge/controllers/__init__.py          |   8 +-
 ..._merge.pull_requests.feedback.template.csv |  11 +-
 .../migrations/15.0.1.10/pre-migration.py     |  11 +
 .../migrations/15.0.1.11/pre-migration.py     | 124 ++++
 runbot_merge/models/commands.py               | 291 +++++---
 runbot_merge/models/project.py                |  29 +-
 .../models/project_freeze/__init__.py         |   5 +-
 runbot_merge/models/pull_requests.py          | 647 +++++++++++-------
 runbot_merge/models/stagings_create.py        |  21 +-
 runbot_merge/tests/test_basic.py              | 319 ++++++---
 runbot_merge/tests/test_multirepo.py          |  55 +-
 runbot_merge/tests/test_oddities.py           |  24 +
 runbot_merge/tests/test_project_toggles.py    |   0
 runbot_merge/views/mergebot.xml               |  74 +-
 27 files changed, 1272 insertions(+), 776 deletions(-)
 create mode 100644 forwardport/migrations/15.0.1.4/pre-migration.py
 create mode 100644 runbot_merge/changelog/2023-12/commands.md
 create mode 100644 runbot_merge/migrations/15.0.1.10/pre-migration.py
 create mode 100644 runbot_merge/migrations/15.0.1.11/pre-migration.py
 create mode 100644 runbot_merge/tests/test_project_toggles.py

diff --git a/forwardport/__manifest__.py b/forwardport/__manifest__.py
index 1f59983e..cb24f840 100644
--- a/forwardport/__manifest__.py
+++ b/forwardport/__manifest__.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 {
     'name': 'forward port bot',
-    'version': '1.3',
+    'version': '1.4',
     'summary': "A port which forward ports successful PRs.",
     'depends': ['runbot_merge'],
     'data': [
diff --git a/forwardport/data/views.xml b/forwardport/data/views.xml
index 4536b79f..01af17e2 100644
--- a/forwardport/data/views.xml
+++ b/forwardport/data/views.xml
@@ -176,7 +176,6 @@
                     </group>
                     <group>
                         <field string="Bot Name" name="fp_github_name" readonly="0"/>
-                        <field string="Bot Email" name="fp_github_email" readonly="0"/>
                     </group>
                 </group>
             </xpath>
@@ -200,37 +199,4 @@
         </field>
     </record>
 
-    <record model="ir.ui.view" id="pr">
-        <field name="name">Show forwardport PR fields</field>
-        <field name="inherit_id" ref="runbot_merge.runbot_merge_form_prs"/>
-        <field name="model">runbot_merge.pull_requests</field>
-        <field name="arch" type="xml">
-            <xpath expr="//field[@name='state']" position="after">
-                <field name="merge_date" attrs="{'invisible': [('state', '!=', 'merged')]}"/>
-            </xpath>
-            <xpath expr="//sheet/group[2]" position="after">
-                <separator string="Forward Port" attrs="{'invisible': [('source_id', '=', False)]}"/>
-                <group attrs="{'invisible': [('source_id', '!=', False)]}">
-                    <group>
-                        <field string="Policy" name="fw_policy"/>
-                    </group>
-                </group>
-                <group attrs="{'invisible': [('source_id', '=', False)]}">
-                    <group>
-                        <field string="Original PR" name="source_id"/>
-                    </group>
-                    <group attrs="{'invisible': [('parent_id', '=', False)]}">
-                        <field name="parent_id"/>
-                    </group>
-                    <group colspan="4" attrs="{'invisible': [('parent_id', '!=', False)]}">
-                        <field string="Detached because" name="detach_reason" readonly="1"/>
-                    </group>
-                    <group>
-                        <field string="Forward ported up to" name="limit_id"/>
-                    </group>
-                </group>
-            </xpath>
-        </field>
-    </record>
-
 </odoo>
diff --git a/forwardport/migrations/15.0.1.4/pre-migration.py b/forwardport/migrations/15.0.1.4/pre-migration.py
new file mode 100644
index 00000000..b1084e5b
--- /dev/null
+++ b/forwardport/migrations/15.0.1.4/pre-migration.py
@@ -0,0 +1,7 @@
+def migrate(cr, version):
+    cr.execute("ALTER TABLE runbot_merge_project DROP COLUMN IF EXISTS fp_github_email")
+    cr.execute("""
+    ALTER TABLE runbot_merge_branch
+        DROP COLUMN IF EXISTS fp_sequence,
+        DROP COLUMN IF EXISTS fp_target
+    """)
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index eb2b36b0..7b856893 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -24,7 +24,6 @@ import re
 import subprocess
 import tempfile
 import typing
-from functools import reduce
 from operator import itemgetter
 from pathlib import Path
 
@@ -194,62 +193,7 @@ class PullRequests(models.Model):
     head: str
     state: str
 
-    statuses = fields.Text(recursive=True)
-
-    limit_id = fields.Many2one('runbot_merge.branch', help="Up to which branch should this PR be forward-ported")
-
-    parent_id = fields.Many2one(
-        'runbot_merge.pull_requests', index=True,
-        help="a PR with a parent is an automatic forward port"
-    )
-    root_id = fields.Many2one('runbot_merge.pull_requests', compute='_compute_root', recursive=True)
-    source_id = fields.Many2one('runbot_merge.pull_requests', index=True, help="the original source of this FP even if parents were detached along the way")
-    forwardport_ids = fields.One2many('runbot_merge.pull_requests', 'source_id')
     reminder_backoff_factor = fields.Integer(default=-4, group_operator=None)
-    merge_date = fields.Datetime()
-
-    detach_reason = fields.Char()
-
-    fw_policy = fields.Selection([
-        ('ci', "Normal"),
-        ('skipci', "Skip CI"),
-        # ('skipmerge', "Skip merge"),
-    ], required=True, default="ci")
-
-    _sql_constraints = [(
-        'fw_constraint',
-        'check(source_id is null or num_nonnulls(parent_id, detach_reason) = 1)',
-        "fw PRs must either be attached or have a reason for being detached",
-    )]
-
-    refname = fields.Char(compute='_compute_refname')
-    @api.depends('label')
-    def _compute_refname(self):
-        for pr in self:
-            pr.refname = pr.label.split(':', 1)[-1]
-
-    ping = fields.Char(recursive=True)
-
-    @api.depends('source_id.author.github_login', 'source_id.reviewed_by.github_login')
-    def _compute_ping(self):
-        """For forward-port PRs (PRs with a source) the author is the PR bot, so
-        we want to ignore that and use the author & reviewer of the original PR
-        """
-        source = self.source_id
-        if not source:
-            return super()._compute_ping()
-
-        for pr in self:
-            s = ' '.join(
-                f'@{p.github_login}'
-                for p in source.author | source.reviewed_by | self.reviewed_by
-            )
-            pr.ping = s and (s + ' ')
-
-    @api.depends('parent_id.root_id')
-    def _compute_root(self):
-        for p in self:
-            p.root_id = reduce(lambda _, p: p, self._iter_ancestors())
 
     @api.model_create_single
     def create(self, vals):
@@ -269,8 +213,6 @@ class PullRequests(models.Model):
             )[-1].id
         if vals.get('parent_id') and 'source_id' not in vals:
             vals['source_id'] = self.browse(vals['parent_id']).root_id.id
-        if vals.get('state') == 'merged':
-            vals['merge_date'] = fields.Datetime.now()
         return super().create(vals)
 
     def write(self, vals):
@@ -302,8 +244,6 @@ class PullRequests(models.Model):
         if vals.get('parent_id') and 'source_id' not in vals:
             parent = self.browse(vals['parent_id'])
             vals['source_id'] = (parent.source_id or parent).id
-        if vals.get('state') == 'merged':
-            vals['merge_date'] = fields.Datetime.now()
         r = super().write(vals)
         if self.env.context.get('forwardport_detach_warn', True):
             for p, parent in with_parents.items():
@@ -322,14 +262,7 @@ class PullRequests(models.Model):
                         token_field='fp_github_token',
                         format_args={'pr': parent, 'child': p},
                     )
-        for p in closed_fp.filtered(lambda p: p.state != 'closed'):
-            self.env.ref('runbot_merge.forwardport.reopen.detached')._send(
-                repository=p.repository,
-                pull_request=p.number,
-                token_field='fp_github_token',
-                format_args={'pr': p},
-            )
-        if vals.get('state') == 'merged':
+        if vals.get('merge_date'):
             self.env['forwardport.branch_remover'].create([
                 {'pr_id': p.id}
                 for p in self
@@ -570,25 +503,6 @@ class PullRequests(models.Model):
         }
         return sorted(commits, key=lambda c: idx[c['sha']])
 
-    def _iter_ancestors(self):
-        while self:
-            yield self
-            self = self.parent_id
-
-    def _iter_descendants(self):
-        pr = self
-        while pr := self.search([('parent_id', '=', pr.id)]):
-            yield pr
-
-    @api.depends('parent_id.statuses')
-    def _compute_statuses(self):
-        super()._compute_statuses()
-
-    def _get_overrides(self):
-        # NB: assumes _get_overrides always returns an "owned" dict which we can modify
-        p = self.parent_id._get_overrides() if self.parent_id else {}
-        p.update(super()._get_overrides())
-        return p
 
     def _port_forward(self):
         if not self:
@@ -713,7 +627,6 @@ class PullRequests(models.Model):
             new_batch |= new_pr
 
             # allows PR author to close or skipci
-            source.delegates |= source.author
             new_pr.write({
                 'merge_method': pr.merge_method,
                 'source_id': source.id,
@@ -722,9 +635,6 @@ class PullRequests(models.Model):
                 'detach_reason': "conflicts: {}".format(
                     f'\n{conflicts[pr]}\n{conflicts[pr]}'.strip()
                 ) if has_conflicts else None,
-                # Copy author & delegates of source as well as delegates of
-                # previous so they can r+ the new forward ports.
-                'delegates': [(6, False, (source.delegates | pr.delegates).ids)]
             })
             if has_conflicts and pr.parent_id and pr.state not in ('merged', 'closed'):
                 self.env.ref('runbot_merge.forwardport.failure.conflict')._send(
@@ -809,9 +719,8 @@ class PullRequests(models.Model):
             'prs': [(6, 0, new_batch.ids)],
             'active': not has_conflicts,
         })
-        # if we're not waiting for CI, schedule followup immediately
-        if any(p.source_id.fw_policy == 'skipci' for p in b.prs):
-            b.prs[0]._schedule_fp_followup()
+        # try to schedule followup
+        new_batch[0]._schedule_fp_followup()
         return b
 
     def _create_fp_branch(self, target_branch, fp_branch_name, cleanup):
@@ -869,7 +778,7 @@ class PullRequests(models.Model):
         # add target remote
         working_copy.remote(
             'add', 'target',
-            'https://{p.fp_github_name}:{p.fp_github_token}@github.com/{r.fp_remote_target}'.format(
+            'https://{p.fp_github_token}@github.com/{r.fp_remote_target}'.format(
                 r=self.repository,
                 p=project_id
             )
diff --git a/forwardport/tests/test_conflicts.py b/forwardport/tests/test_conflicts.py
index eeeaf7ea..4cad2c8c 100644
--- a/forwardport/tests/test_conflicts.py
+++ b/forwardport/tests/test_conflicts.py
@@ -93,10 +93,6 @@ In the former case, you may want to edit this PR message as well\.
 More info at https://github\.com/odoo/odoo/wiki/Mergebot#forward-port
 ''', re.DOTALL))
     ]
-    with prod:
-        prc.post_comment(f'@{project.fp_github_name} r+', config['role_reviewer']['token'])
-    env.run_crons()
-    assert prc_id.state == 'opened', "approving via fw should not work on a conflict"
 
     prb = prod.get_pr(prb_id.number)
     assert prb.comments == [
@@ -108,13 +104,12 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 '''),
         (users['user'], """@%s @%s the next pull request (%s) is in conflict. \
 You can merge the chain up to here by saying
-> @%s r+
+> @hansen r+
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 """ % (
             users['user'], users['reviewer'],
             prc_id.display_name,
-            project.fp_github_name
         ))
     ]
 
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index a2c174c5..a75d586e 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -11,7 +11,6 @@ from utils import seen, Commit, make_basic, to_pr
 ])
 def test_configure_fp_limit(env, config, make_repo, source, limit, count):
     prod, other = make_basic(env, config, make_repo)
-    bot_name = env['runbot_merge.project'].search([]).fp_github_name
     with prod:
         [c] = prod.make_commits(
             source, Commit('c', tree={'f': 'g'}),
@@ -20,7 +19,7 @@ def test_configure_fp_limit(env, config, make_repo, source, limit, count):
         pr = prod.make_pr(target=source, head='branch')
         prod.post_status(c, 'success', 'legal/cla')
         prod.post_status(c, 'success', 'ci/runbot')
-        pr.post_comment(f'hansen r+\n{bot_name} up to {limit}', config['role_reviewer']['token'])
+        pr.post_comment(f'hansen r+ up to {limit}', config['role_reviewer']['token'])
     env.run_crons()
     with prod:
         prod.post_status(f'staging.{source}', 'success', 'legal/cla')
@@ -38,14 +37,13 @@ def test_ignore(env, config, make_repo):
     to target
     """
     prod, other = make_basic(env, config, make_repo)
-    bot_name = env['runbot_merge.project'].search([]).fp_github_name
     branch_a = env['runbot_merge.branch'].search([('name', '=', 'a')])
     with prod:
         [c] = prod.make_commits('a', Commit('c', tree={'0': '0'}), ref='heads/mybranch')
         pr = prod.make_pr(target='a', head='mybranch')
         prod.post_status(c, 'success', 'legal/cla')
         prod.post_status(c, 'success', 'ci/runbot')
-        pr.post_comment('hansen r+\n%s ignore' % bot_name, config['role_reviewer']['token'])
+        pr.post_comment('hansen r+ ignore', config['role_reviewer']['token'])
     env.run_crons()
     pr_id = env['runbot_merge.pull_requests'].search([('number', '=', pr.number)])
     assert pr_id.limit_id == branch_a
@@ -67,13 +65,12 @@ def test_disable(env, config, make_repo, users):
     """
     prod, other = make_basic(env, config, make_repo)
     project = env['runbot_merge.project'].search([])
-    bot_name = project.fp_github_name
     with prod:
         [c] = prod.make_commits('a', Commit('c 0', tree={'0': '0'}), ref='heads/branch0')
         pr = prod.make_pr(target='a', head='branch0')
         prod.post_status(c, 'success', 'legal/cla')
         prod.post_status(c, 'success', 'ci/runbot')
-        pr.post_comment('hansen r+\n%s up to b' % bot_name, config['role_reviewer']['token'])
+        pr.post_comment('hansen r+ up to b', config['role_reviewer']['token'])
 
         [c] = prod.make_commits('a', Commit('c 1', tree={'1': '1'}), ref='heads/branch1')
         pr = prod.make_pr(target='a', head='branch1')
@@ -94,30 +91,28 @@ def test_disable(env, config, make_repo, users):
     assert p.parent_id == _1
     assert p.target.name == 'c'
 
-    project.fp_github_token = config['role_other']['token']
-    bot_name = project.fp_github_name
     with prod:
         [c] = prod.make_commits('a', Commit('c 2', tree={'2': '2'}), ref='heads/branch2')
         pr = prod.make_pr(target='a', head='branch2')
         prod.post_status(c, 'success', 'legal/cla')
         prod.post_status(c, 'success', 'ci/runbot')
-        pr.post_comment(f'hansen r+\n{bot_name} up to', config['role_reviewer']['token'])
-        pr.post_comment(f'{bot_name} up to b', config['role_reviewer']['token'])
-        pr.post_comment(f'{bot_name} up to foo', config['role_reviewer']['token'])
-        pr.post_comment(f'{bot_name} up to c', config['role_reviewer']['token'])
+        pr.post_comment('hansen r+ up to', config['role_reviewer']['token'])
+        pr.post_comment('hansen up to b', config['role_reviewer']['token'])
+        pr.post_comment('hansen up to foo', config['role_reviewer']['token'])
+        pr.post_comment('hansen up to c', config['role_reviewer']['token'])
     env.run_crons()
 
     # use a set because git webhooks delays might lead to mis-ordered
     # responses and we don't care that much
     assert set(pr.comments) == {
-        (users['reviewer'], f"hansen r+\n{bot_name} up to"),
-        (users['other'], "@{reviewer} please provide a branch to forward-port to.".format_map(users)),
-        (users['reviewer'], f"{bot_name} up to b"),
-        (users['other'], "@{reviewer} branch 'b' is disabled, it can't be used as a forward port target.".format_map(users)),
-        (users['reviewer'], f"{bot_name} up to foo"),
-        (users['other'], "@{reviewer} there is no branch 'foo', it can't be used as a forward port target.".format_map(users)),
-        (users['reviewer'], f"{bot_name} up to c"),
-        (users['other'], "Forward-porting to 'c'."),
+        (users['reviewer'], "hansen r+ up to"),
+        (users['user'], "@{reviewer} please provide a branch to forward-port to.".format_map(users)),
+        (users['reviewer'], "hansen up to b"),
+        (users['user'], "@{reviewer} branch 'b' is disabled, it can't be used as a forward port target.".format_map(users)),
+        (users['reviewer'], "hansen up to foo"),
+        (users['user'], "@{reviewer} there is no branch 'foo', it can't be used as a forward port target.".format_map(users)),
+        (users['reviewer'], "hansen up to c"),
+        (users['user'], "Forward-porting to 'c'."),
         seen(env, pr, users),
     }
 
@@ -127,7 +122,6 @@ def test_limit_after_merge(env, config, make_repo, users):
     reviewer = config['role_reviewer']['token']
     branch_b = env['runbot_merge.branch'].search([('name', '=', 'b')])
     branch_c = env['runbot_merge.branch'].search([('name', '=', 'c')])
-    bot_name = env['runbot_merge.project'].search([]).fp_github_name
     with prod:
         [c] = prod.make_commits('a', Commit('c', tree={'0': '0'}), ref='heads/abranch')
         pr1 = prod.make_pr(target='a', head='abranch')
@@ -145,15 +139,15 @@ def test_limit_after_merge(env, config, make_repo, users):
     assert p1.limit_id == p2.limit_id == branch_c, "check that limit is correctly set"
     pr2 = prod.get_pr(p2.number)
     with prod:
-        pr1.post_comment(bot_name + ' up to b', reviewer)
-        pr2.post_comment(bot_name + ' up to b', reviewer)
+        pr1.post_comment('hansen up to b', reviewer)
+        pr2.post_comment('hansen up to b', reviewer)
     env.run_crons()
 
     assert p1.limit_id == p2.limit_id == branch_b
     assert pr1.comments == [
         (users['reviewer'], "hansen r+"),
         seen(env, pr1, users),
-        (users['reviewer'], f'{bot_name} up to b'),
+        (users['reviewer'], 'hansen up to b'),
         (users['user'], "Forward-porting to 'b'."),
         (users['user'], f"Forward-porting to 'b' (from {p2.display_name})."),
     ]
@@ -164,7 +158,7 @@ This PR targets b and is part of the forward-port chain. Further PRs will be cre
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 """),
-        (users['reviewer'], f'{bot_name} up to b'),
+        (users['reviewer'], 'hansen up to b'),
         (users['user'], f"Forward-porting {p1.display_name} to 'b'."),
     ]
 
@@ -181,16 +175,12 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
     assert p2.source_id == p1
 
     with prod:
-        pr2.post_comment(f'{bot_name} up to c', reviewer)
+        pr2.post_comment('hansen up to c', reviewer)
     env.run_crons()
 
     assert pr2.comments[4:] == [
-        (users['user'], "@%s @%s this PR was modified / updated and has become a normal PR. "
-                   "It should be merged the normal way (via @%s)" % (
-            users['user'], users['reviewer'],
-            p2.repository.project_id.github_prefix
-        )),
-        (users['reviewer'], f'{bot_name} up to c'),
+        (users['user'], f"@{users['user']} @{users['reviewer']} this PR was modified / updated and has become a normal PR. It must be merged directly."),
+        (users['reviewer'], 'hansen up to c'),
         (users['user'], "Forward-porting to 'c'."),
     ]
     with prod:
@@ -207,7 +197,7 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
     assert p3
     pr3 = prod.get_pr(p3.number)
     with prod:
-        pr3.post_comment(f"{bot_name} up to c", reviewer)
+        pr3.post_comment("hansen up to c", reviewer)
     env.run_crons()
     assert pr3.comments == [
         seen(env, pr3, users),
@@ -215,11 +205,11 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 @{users['user']} @{users['reviewer']} this PR targets c and is the last of the forward-port chain.
 
 To merge the full chain, use
-> @{p1.repository.project_id.fp_github_name} r+
+> @hansen r+
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 """),
-        (users['reviewer'], f"{bot_name} up to c"),
+        (users['reviewer'], "hansen up to c"),
         (users['user'], f"Forward-porting {p2.display_name} to 'c'."),
     ]
     # 7 of previous check, plus r+
@@ -268,7 +258,7 @@ def test_post_merge(
     from_id = PRs.search(update_from(source.id))
     from_ = prod.get_pr(from_id.number)
     with prod:
-        from_.post_comment(f'{project.fp_github_name} up to {limit}', reviewer)
+        from_.post_comment(f'hansen up to {limit}', reviewer)
     env.run_crons()
 
     # there should always be a comment on the source and root indicating how
@@ -314,7 +304,7 @@ def test_resume_fw(env, post_merge, users, config, branches, mode):
     # fetch source PR
     [source] = PRs.search([('source_id', '=', False)])
     with prod:
-        prod.get_pr(source.number).post_comment(f'{project.fp_github_name} up to 5', reviewer)
+        prod.get_pr(source.number).post_comment('hansen up to 5', reviewer)
     # validate the forward ports for "child", "root", and "parent" so "current"
     # exists and we have one more target
     for branch in map(str, range(2, 5+1)):
@@ -336,12 +326,11 @@ def test_resume_fw(env, post_merge, users, config, branches, mode):
         numbers = range(5 if mode == 'mergetip' else 2, 5 + 1)
         with prod:
             for number in numbers:
-                prod.get_pr(number).post_comment(f'{project.github_prefix} r+', reviewer)
+                prod.get_pr(number).post_comment('hansen r+', reviewer)
         env.run_crons()
         with prod:
             for target in numbers:
                 pr = PRs.search([('target.name', '=', str(target))])
-                print(pr.display_name, pr.state, pr.staging_id)
                 prod.post_status(f'staging.{target}', 'success')
         env.run_crons()
         for number in numbers:
@@ -349,7 +338,7 @@ def test_resume_fw(env, post_merge, users, config, branches, mode):
 
     from_ = prod.get_pr(source.number)
     with prod:
-        from_.post_comment(f'{project.fp_github_name} up to 6', reviewer)
+        from_.post_comment('hansen up to 6', reviewer)
     env.run_crons()
 
     if mode == 'failbump':
@@ -419,7 +408,6 @@ def post_merge(env, config, users, make_repo, branches):
         'github_prefix': 'hansen',
         'fp_github_token': config['github']['token'],
         'fp_github_name': 'herbert',
-        'fp_github_email': 'hb@example.com',
         'branch_ids': [
             (0, 0, {'name': str(i), 'sequence': 1000 - (i * 10)})
             for i in branches
@@ -439,7 +427,6 @@ def post_merge(env, config, users, make_repo, branches):
         'review_rights': [(0, 0, {'repository_id': proj.repo_ids.id, 'review': True})]
     })
 
-    mbot = proj.github_prefix
     reviewer = config['role_reviewer']['token']
     # merge the source PR
     source_target = str(branches[0])
@@ -448,7 +435,7 @@ def post_merge(env, config, users, make_repo, branches):
         pr1 = prod.make_pr(target=source_target, head=c, title="a title")
 
         prod.post_status(c, 'success')
-        pr1.post_comment(f'{mbot} r+', reviewer)
+        pr1.post_comment('hansen r+', reviewer)
     env.run_crons()
     with prod:
         prod.post_status(f'staging.{source_target}', 'success')
diff --git a/forwardport/tests/test_overrides.py b/forwardport/tests/test_overrides.py
index e8a18d35..eb5acf12 100644
--- a/forwardport/tests/test_overrides.py
+++ b/forwardport/tests/test_overrides.py
@@ -12,39 +12,45 @@ def test_override_inherited(env, config, make_repo, users):
     """
     repo, other = make_basic(env, config, make_repo)
     project = env['runbot_merge.project'].search([])
+    project.repo_ids.status_ids = [(5, 0, 0), (0, 0, {'context': 'default'})]
     env['res.partner'].search([('github_login', '=', users['reviewer'])])\
         .write({'override_rights': [(0, 0, {
             'repository_id': project.repo_ids.id,
-            'context': 'ci/runbot',
+            'context': 'default',
         })]})
 
     with repo:
-        repo.make_commits('a', Commit('C', tree={'a': '0'}), ref='heads/change')
+        repo.make_commits('a', Commit('pr 1', tree={'a': '0'}), ref='heads/change')
         pr = repo.make_pr(target='a', head='change')
-        repo.post_status('change', 'success', 'legal/cla')
-        pr.post_comment('hansen r+ override=ci/runbot', config['role_reviewer']['token'])
+        pr.post_comment('hansen r+ override=default', config['role_reviewer']['token'])
     env.run_crons()
 
     original = env['runbot_merge.pull_requests'].search([('repository.name', '=', repo.name), ('number', '=', pr.number)])
     assert original.state == 'ready'
+    assert original.limit_id.name == 'c'
 
     with repo:
-        repo.post_status('staging.a', 'success', 'legal/cla')
-        repo.post_status('staging.a', 'success', 'ci/runbot')
+        repo.post_status('staging.a', 'success')
     env.run_crons()
 
-    pr0_id, pr1_id = env['runbot_merge.pull_requests'].search([], order='number')
+    pr0_id, pr1_id, pr2_id = env['runbot_merge.pull_requests'].search([], order='number')
     assert pr0_id == original
-    assert pr1_id.parent_id, pr0_id
+    assert pr0_id.target.name == 'a'
 
-    with repo:
-        repo.post_status(pr1_id.head, 'success', 'legal/cla')
-    env.run_crons()
+    assert pr1_id.parent_id == pr0_id
+    assert pr1_id.number == 2
+    assert pr1_id.target.name == 'b'
     assert pr1_id.state == 'validated'
-    assert statuses(pr1_id) == {'ci/runbot': 'success', 'legal/cla': 'success'}
+    assert statuses(pr1_id) == {'default': 'success'}
+
+    assert pr2_id.parent_id == pr1_id
+    assert pr2_id.target.name == 'c'
+    assert pr2_id.state == 'validated'
+    assert statuses(pr2_id) == {'default': 'success'}
 
     # now we edit the child PR
-    pr_repo, pr_ref = repo.get_pr(pr1_id.number).branch
+    pr1 = repo.get_pr(pr1_id.number)
+    pr_repo, pr_ref = pr1.branch
     with pr_repo:
         pr_repo.make_commits(
             pr1_id.target.name,
@@ -56,6 +62,12 @@ def test_override_inherited(env, config, make_repo, users):
     assert pr1_id.state == 'opened'
     assert not pr1_id.parent_id
     assert statuses(pr1_id) == {}, "should not have any status left"
+    assert statuses(pr2_id) == {}
+
+    with repo:
+        pr1.post_comment('hansen override=default', config['role_reviewer']['token'])
+    assert statuses(pr1_id) == {'default': 'success'}
+    assert statuses(pr2_id) == {'default': 'success'}
 
 def test_override_combination(env, config, make_repo, users):
     """ A forwardport should inherit its parents' overrides, until it's edited.
diff --git a/forwardport/tests/test_simple.py b/forwardport/tests/test_simple.py
index 6d398f64..cc8bb3c1 100644
--- a/forwardport/tests/test_simple.py
+++ b/forwardport/tests/test_simple.py
@@ -161,20 +161,19 @@ def test_straightforward_flow(env, config, make_repo, users):
 * %s
 
 To merge the full chain, use
-> @%s r+
+> @hansen r+
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 """ % (
             users['other'], users['reviewer'],
             pr1.display_name,
-            project.fp_github_name
     )),
     ]
     with prod:
         prod.post_status(pr2.head, 'success', 'ci/runbot')
         prod.post_status(pr2.head, 'success', 'legal/cla')
 
-        pr2_remote.post_comment('%s r+' % project.fp_github_name, config['role_reviewer']['token'])
+        pr2_remote.post_comment('hansen r+', config['role_reviewer']['token'])
 
     env.run_crons()
 
@@ -317,7 +316,6 @@ def test_empty(env, config, make_repo, users):
     project = env['runbot_merge.project'].search([])
     project.write({
         'fp_github_name': False,
-        'fp_github_email': False,
         'fp_github_token': config['role_other']['token'],
     })
     assert project.fp_github_name == users['other']
@@ -493,7 +491,7 @@ def test_access_rights(env, config, make_repo, users, author, reviewer, delegate
         prod.post_status(pr2.head, 'success', 'ci/runbot')
         prod.post_status(pr2.head, 'success', 'legal/cla')
         prod.get_pr(pr2.number).post_comment(
-            '%s r+' % project.fp_github_name,
+            'hansen r+',
             token=config['role_' + reviewer]['token']
         )
     env.run_crons()
@@ -587,10 +585,10 @@ def test_delegate_fw(env, config, make_repo, users):
         (users['user'], '''@{self_reviewer} @{reviewer} this PR targets c and is the last of the forward-port chain.
 
 To merge the full chain, use
-> @{bot} r+
+> @hansen r+
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
-'''.format(bot=pr1_id.repository.project_id.fp_github_name, **users)),
+'''.format_map(users)),
         (users['other'], 'hansen r+')
     ]
 
@@ -630,7 +628,7 @@ def test_redundant_approval(env, config, make_repo, users):
     with prod:
         pr1.post_comment('hansen r+', config['role_reviewer']['token'])
     with prod:
-        pr2.post_comment(f'{project.fp_github_name} r+', config['role_reviewer']['token'])
+        pr2.post_comment('hansen r+', config['role_reviewer']['token'])
     env.run_crons()
 
     assert pr1.comments == [
@@ -742,7 +740,7 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
     # ok main1 PRs
     with main1:
         validate_all([main1], [pr1c.head])
-        main1.get_pr(pr1c.number).post_comment('%s r+' % project.fp_github_name, config['role_reviewer']['token'])
+        main1.get_pr(pr1c.number).post_comment('hansen r+', config['role_reviewer']['token'])
     env.run_crons()
 
     # check that the main1 PRs are ready but blocked on the main2 PRs
@@ -754,7 +752,7 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
     # ok main2 PRs
     with main2:
         validate_all([main2], [pr2c.head])
-        main2.get_pr(pr2c.number).post_comment('%s r+' % project.fp_github_name, config['role_reviewer']['token'])
+        main2.get_pr(pr2c.number).post_comment('hansen r+', config['role_reviewer']['token'])
     env.run_crons()
 
     env['runbot_merge.stagings'].search([]).mapped('target.display_name')
@@ -862,27 +860,8 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
         with prod:
             pr1.open()
         assert pr1_id.state == 'validated'
-        env.run_crons()
-        assert pr1.comments[-1] == (
-            users['user'],
-            "@{} @{} this PR was closed then reopened. "
-            "It should be merged the normal way (via @{})".format(
-                users['user'],
-                users['reviewer'],
-                project.github_prefix,
-            )
-        )
-
-        with prod:
-            pr1.post_comment(f'{project.fp_github_name} r+', config['role_reviewer']['token'])
-        env.run_crons()
-        assert pr1.comments[-1] == (
-            users['user'],
-            "@{} I can only do this on unmodified forward-port PRs, ask {}.".format(
-                users['reviewer'],
-                project.github_prefix,
-            ),
-        )
+        assert not pr1_id.parent_id
+        assert not pr2_id.parent_id
 
     def test_close_disabled(self, env, make_repo, users, config):
         """ If an fwport's target is disabled and its branch is closed, it
@@ -937,7 +916,7 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 * {pr2_id.display_name}
 
 To merge the full chain, use
-> @herbert r+
+> @hansen r+
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 """.format(pr2_id=pr2_id, **users)),
@@ -1063,13 +1042,13 @@ class TestRecognizeCommands:
             ('number', '=', pr.number),
         ])
 
+    # FIXME: remove / merge into mergebot tests
     def test_botname_casing(self, env, config, make_repo):
         """ Test that the botname is case-insensitive as people might write
         bot names capitalised or titlecased or uppercased or whatever
         """
         repo, pr, pr_id = self.make_pr(env, config, make_repo)
         assert pr_id.state == 'opened'
-        botname = env['runbot_merge.project'].search([]).fp_github_name
         [a] = env['runbot_merge.branch'].search([
             ('name', '=', 'a')
         ])
@@ -1078,27 +1057,27 @@ class TestRecognizeCommands:
         ])
 
         names = [
-            botname,
-            botname.upper(),
-            botname.capitalize(),
-            sPeNgBaB(botname),
+            "hansen",
+            "HANSEN",
+            "Hansen",
+            sPeNgBaB("hansen"),
         ]
 
         for n in names:
             assert pr_id.limit_id == c
             with repo:
-                pr.post_comment('@%s up to a' % n, config['role_reviewer']['token'])
+                pr.post_comment(f'@{n} up to a', config['role_reviewer']['token'])
             assert pr_id.limit_id == a
             # reset state
             pr_id.write({'limit_id': c.id})
 
+    # FIXME: remove / merge into mergebot tests
     @pytest.mark.parametrize('indent', ['', '\N{SPACE}', '\N{SPACE}'*4, '\N{TAB}'])
     def test_botname_indented(self, env, config, make_repo, indent):
         """ matching botname should ignore leading whitespaces
         """
         repo, pr, pr_id = self.make_pr(env, config, make_repo)
         assert pr_id.state == 'opened'
-        botname = env['runbot_merge.project'].search([]).fp_github_name
         [a] = env['runbot_merge.branch'].search([
             ('name', '=', 'a')
         ])
@@ -1108,5 +1087,5 @@ class TestRecognizeCommands:
 
         assert pr_id.limit_id == c
         with repo:
-            pr.post_comment('%s@%s up to a' % (indent, botname), config['role_reviewer']['token'])
+            pr.post_comment(f'{indent}@hansen up to a', config['role_reviewer']['token'])
         assert pr_id.limit_id == a
diff --git a/forwardport/tests/test_updates.py b/forwardport/tests/test_updates.py
index 1f5269df..3a6b4a11 100644
--- a/forwardport/tests/test_updates.py
+++ b/forwardport/tests/test_updates.py
@@ -25,7 +25,7 @@ def test_update_pr(env, config, make_repo, users, merge_parent) -> None:
     })
     with prod:
         prod.make_commits('c', Commit('1111', tree={'i': 'a'}), ref='heads/d')
-    
+
     with prod:
         [p_1] = prod.make_commits(
             'a',
@@ -108,15 +108,6 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 
     assert pr1_id.head == new_c != pr1_head, "the FP PR should be updated"
     assert not pr1_id.parent_id, "the FP PR should be detached from the original"
-    assert pr1_remote.comments == [
-        seen(env, pr1_remote, users),
-        fp_intermediate, ci_warning, ci_warning,
-        (users['user'], "@%s @%s this PR was modified / updated and has become a normal PR. "
-                        "It should be merged the normal way (via @%s)" % (
-            users['user'], users['reviewer'],
-            pr1_id.repository.project_id.github_prefix
-        )),
-    ], "users should be warned that the PR has become non-FP"
     # NOTE: should the followup PR wait for pr1 CI or not?
     assert pr2_id.head != pr2_head
     assert pr2_id.parent_id == pr1_id, "the followup PR should still be linked"
@@ -132,7 +123,7 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
         'h': 'a',
         'x': '5'
     }, "the followup FP should also have the update"
-    
+
     with prod:
         prod.post_status(pr2_id.head, 'success', 'ci/runbot')
         prod.post_status(pr2_id.head, 'success', 'legal/cla')
@@ -155,7 +146,7 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
     pr3_id.write({'parent_id': False, 'detach_reason': "testing"})
     # pump feedback messages
     env.run_crons()
-    
+
     pr3 = prod.get_pr(pr3_id.number)
     assert pr3.comments == [
         seen(env, pr3, users),
@@ -164,14 +155,13 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 * {pr2_id.display_name}
 
 To merge the full chain, use
-> @{pr3_id.repository.project_id.fp_github_name} r+
+> @hansen r+
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 """),
         (users['user'], f"@{users['user']} @{users['reviewer']} this PR was "
                         f"modified / updated and has become a normal PR. It "
-                        f"should be merged the normal way "
-                        f"(via @{pr3_id.repository.project_id.github_prefix})"
+                        f"must be merged directly."
         )
     ]
 
@@ -197,7 +187,6 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
                             f"won't cross."),
         ]
 
-
 def test_update_merged(env, make_repo, config, users):
     """ Strange things happen when an FP gets closed / merged but then its
     parent is modified and the forwardport tries to update the (now merged)
@@ -322,7 +311,6 @@ def test_duplicate_fw(env, make_repo, setreviewers, config, users):
         'github_prefix': 'hansen',
         'fp_github_token': config['github']['token'],
         'fp_github_name': 'herbert',
-        'fp_github_email': 'hb@example.com',
         'branch_ids': [
             (0, 0, {'name': 'master', 'sequence': 0}),
             (0, 0, {'name': 'v3', 'sequence': 1}),
@@ -377,7 +365,7 @@ def test_duplicate_fw(env, make_repo, setreviewers, config, users):
     with repo:
         repo.make_commits('v2', Commit('c0', tree={'z': 'b'}), ref=prv2.ref, make=False)
     env.run_crons()
-    assert pr_ids.mapped('state') == ['merged', 'opened', 'validated', 'validated']
+    assert pr_ids.mapped('state') == ['merged', 'opened', 'opened', 'opened']
     assert repo.read_tree(repo.commit(prv2_id.head)) == {'f': 'c', 'h': 'a', 'z': 'b'}
     assert repo.read_tree(repo.commit(prv3_id.head)) == {'f': 'd', 'i': 'a', 'z': 'b'}
     assert repo.read_tree(repo.commit(prmaster_id.head)) == {'f': 'e', 'z': 'b'}
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index 1c1fc4ab..a21f086f 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -26,7 +26,6 @@ def make_basic(env, config, make_repo, *, fp_token, fp_remote):
             'github_prefix': 'hansen',
             'fp_github_token': fp_token and config['github']['token'],
             'fp_github_name': 'herbert',
-            'fp_github_email': 'hb@example.com',
             'branch_ids': [
                 (0, 0, {'name': 'a', 'sequence': 2}),
                 (0, 0, {'name': 'b', 'sequence': 1}),
@@ -181,7 +180,7 @@ def test_failed_staging(env, config, make_repo):
     with prod:
         prod.post_status(pr3_id.head, 'success', 'legal/cla')
         prod.post_status(pr3_id.head, 'success', 'ci/runbot')
-        pr3.post_comment('%s r+' % proj.fp_github_name, reviewer)
+        pr3.post_comment('hansen r+', reviewer)
     env.run_crons()
 
     prod.commit('staging.c')
@@ -265,7 +264,6 @@ class TestNotAllBranches:
             'github_prefix': 'hansen',
             'fp_github_token': config['github']['token'],
             'fp_github_name': 'herbert',
-            'fp_github_email': 'hb@example.com',
             'branch_ids': [
                 (0, 0, {'name': 'a', 'sequence': 2}),
                 (0, 0, {'name': 'b', 'sequence': 1}),
@@ -318,7 +316,7 @@ class TestNotAllBranches:
         with a:
             a.post_status(pr2.head, 'success', 'ci/runbot')
             a.get_pr(pr2.number).post_comment(
-                '%s r+' % project.fp_github_name,
+                'hansen r+',
                 config['role_reviewer']['token'])
         env.run_crons()
         assert pr1.staging_id
@@ -357,7 +355,7 @@ class TestNotAllBranches:
         with b:
             b.post_status(pr1.head, 'success', 'ci/runbot')
             b.get_pr(pr1.number).post_comment(
-                '%s r+' % project.fp_github_name,
+                'hansen r+',
                 config['role_reviewer']['token'])
         env.run_crons()
         with a, b:
@@ -580,7 +578,7 @@ def test_new_intermediate_branch(env, config, make_repo):
     with prod, prod2:
         for pr in fps.filtered(lambda p: p.target.name == 'c'):
             get_repo(pr).get_pr(pr.number).post_comment(
-                '%s r+' % project.fp_github_name,
+                'hansen r+',
                 config['role_reviewer']['token'])
     assert all(p.state == 'merged' for p in PRs.browse(sources)),\
         "all sources should be merged"
@@ -627,7 +625,7 @@ def test_author_can_close_via_fwbot(env, config, make_repo):
         pr.open(other_token)
         prod.post_status(c, 'success', 'legal/cla')
         prod.post_status(c, 'success', 'ci/runbot')
-        pr.post_comment('%s close' % project.fp_github_name, other_token)
+        pr.post_comment('hansen close', other_token)
         pr.post_comment('hansen r+', config['role_reviewer']['token'])
     env.run_crons()
     assert pr.state == 'open'
@@ -647,7 +645,7 @@ def test_author_can_close_via_fwbot(env, config, make_repo):
         pr1.close(other_token)
     # use can close via fwbot
     with prod:
-        pr1.post_comment('%s close' % project.fp_github_name, other_token)
+        pr1.post_comment('hansen close', other_token)
     env.run_crons()
     assert pr1.state == 'closed'
     assert pr1_id.state == 'closed'
@@ -660,7 +658,7 @@ def test_skip_ci_all(env, config, make_repo):
         pr = prod.make_pr(target='a', head='change')
         prod.post_status(pr.head, 'success', 'legal/cla')
         prod.post_status(pr.head, 'success', 'ci/runbot')
-        pr.post_comment('%s skipci' % project.fp_github_name, config['role_reviewer']['token'])
+        pr.post_comment('hansen fw=skipci', config['role_reviewer']['token'])
         pr.post_comment('hansen r+', config['role_reviewer']['token'])
     env.run_crons()
     assert env['runbot_merge.pull_requests'].search([
@@ -703,8 +701,8 @@ def test_skip_ci_next(env, config, make_repo):
     pr0_id, pr1_id = env['runbot_merge.pull_requests'].search([], order='number')
     with prod:
         prod.get_pr(pr1_id.number).post_comment(
-            '%s skipci' % project.fp_github_name,
-            config['role_user']['token']
+            'hansen fw=skipci',
+            config['role_reviewer']['token']
         )
     assert pr0_id.fw_policy == 'skipci'
     env.run_crons()
diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index 22f69ec5..5da2a744 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -75,7 +75,6 @@ def make_basic(env, config, make_repo, *, reponame='proj', project_name='myproje
             'github_prefix': 'hansen',
             'fp_github_token': config['github']['token'],
             'fp_github_name': 'herbert',
-            'fp_github_email': 'hb@example.com',
             'branch_ids': [
                 (0, 0, {'name': 'a', 'sequence': 100}),
                 (0, 0, {'name': 'b', 'sequence': 80}),
@@ -145,3 +144,7 @@ def part_of(label, pr_id, *, separator='\n\n'):
     """ Adds the "part-of" pseudo-header in the footer.
     """
     return f'{label}{separator}Part-of: {pr_id.display_name}'
+
+def ensure_one(records):
+    assert len(records) == 1
+    return records
diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py
index d36aea6e..bcb702b6 100644
--- a/runbot_merge/__manifest__.py
+++ b/runbot_merge/__manifest__.py
@@ -1,7 +1,7 @@
 {
     'name': 'merge bot',
-    'version': '1.9',
-    'depends': ['contacts', 'website'],
+    'version': '1.11',
+    'depends': ['contacts', 'mail', 'website'],
     'data': [
         'security/security.xml',
         'security/ir.model.access.csv',
diff --git a/runbot_merge/changelog/2023-12/commands.md b/runbot_merge/changelog/2023-12/commands.md
new file mode 100644
index 00000000..1d054179
--- /dev/null
+++ b/runbot_merge/changelog/2023-12/commands.md
@@ -0,0 +1,57 @@
+CHG: complete rework of the commands system
+
+# fun is dead: strict commands parsing
+
+Historically the bots would apply whatever looked like a command and ignore the
+rest. This led to people sending novels to the bot, then being surprised the bot
+found a command in the mess.
+
+The bots now ignore all lines which contain any non-command. Example:
+
+> @robodoo r+ when green darling
+
+Previously, the bot would apply the `r+` and ignore the rest. Now the bot will
+ignore everything and reply with
+
+> unknown command "when"
+
+# fwbot is dead
+
+The mergebot (@robodoo) is now responsible for the old fwbot commands:
+
+- close, ignore, up to, ... work as they ever did, just with robodoo
+- `robodoo r+` now approves the parents if the current PR a forward port
+  - a specific PR can be approved even in forward ports by providing its number
+    e.g. `robodoo r=45328` will approve just PR 45328, if that is the PR the
+    comment is being posted on or one of its parents
+  - the approval of forward ports won't skip over un-approvable PRs anymore
+  - the rights of the original author have been restricted slightly: they can
+    only approve the direct descendents of merged PRs, so if one of the parents
+    has been modified and is not merged yet, the original author can't approve,
+    nor can they approve the modified PR, or a conflicting PR which has to get
+    fixed (?)
+
+# no more p=<number>
+
+The old priorities command was a tangle of multiple concerns, not all of which
+were always desired or applicable. These tangles have been split along their
+various axis.
+
+# listing
+
+The new commands are:
+
+- `default`, sets the staging priority back to the default
+- `priority`, sets the staging priority to elevated, on staging these PRs are
+  staged first, then the `normal` PRs are added
+- `alone`, sets the staging priority to high, these PRs are staged before
+  considering splits, and only `alone` PRs are staged together even if the batch
+  is not full
+- `fw=default`, processes forward ports normally
+- `fw=skipci`, once the current PR has been merged creates all the forward ports
+  without waiting for each to have valid statuses
+- `fw=skipmerge`, immediately create all forward ports even if the base pull
+  request has not even been merged yet
+- `skipchecks`, makes the entire batch (target PR and any linked PR) immediately 
+  ready, bypassing statuses and reviews
+- `cancel`, cancels the staging on the target branch, if any
diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index afbe92d6..006f441e 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -288,7 +288,8 @@ def handle_pr(env, event):
         )
 
         pr_obj.write({
-            'state': 'opened',
+            'reviewed_by': False,
+            'error': False,
             'head': pr['head']['sha'],
             'squash': pr['commits'] == 1,
         })
@@ -327,11 +328,10 @@ def handle_pr(env, event):
                 close=True,
                 message=env.ref('runbot_merge.handle.pr.merged')._format(event=event),
             )
-
-        if pr_obj.state == 'closed':
+        elif pr_obj.closed:
             _logger.info('%s reopening %s', event['sender']['login'], pr_obj.display_name)
             pr_obj.write({
-                'state': 'opened',
+                'closed': False,
                 # updating the head triggers a revalidation
                 'head': pr['head']['sha'],
                 'squash': pr['commits'] == 1,
diff --git a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
index bbe809ff..4dd5678f 100644
--- a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
+++ b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
@@ -40,7 +40,7 @@ runbot_merge.command.approve.failure,@{user} you may want to rebuild or fix this
 
 user: github login of comment sender
 pr: pr object to which the command was sent"
-runbot_merge.command.unapprove.p0,"PR priority reset to 1, as pull requests with priority 0 ignore review state.","Responds to r- of pr in p=0.
+runbot_merge.command.unapprove.p0,"Skipchecks removed due to r-.","Responds to r- of pr in skipchecks.
 
 user: github login of comment sender
 pr: pr object to which the command was sent"
@@ -107,16 +107,13 @@ pr: PR where update followup conflict happened
 previous: parent PR which triggered the followup
 stdout: markdown-formatted stdout of git, if any
 stderr: markdown-formatted stderr of git, if any"
-runbot_merge.forwardport.update.detached,{pr.ping}this PR was modified / updated and has become a normal PR. It should be merged the normal way (via @{pr.repository.project_id.github_prefix}),"Comment when a forwardport PR gets updated, documents that the PR now needs to be merged the “normal” way.
+runbot_merge.forwardport.update.detached,{pr.ping}this PR was modified / updated and has become a normal PR. It must be merged directly.,"Comment when a forwardport PR gets updated, documents that the PR now needs to be merged the “normal” way.
 
 pr: the pr in question "
 runbot_merge.forwardport.update.parent,{pr.ping}child PR {child.display_name} was modified / updated and has become a normal PR. This PR (and any of its parents) will need to be merged independently as approvals won't cross.,"Sent to an open PR when its direct child has been detached.
 
 pr: the pr
 child: its detached child"
-runbot_merge.forwardport.reopen.detached,{pr.ping}this PR was closed then reopened. It should be merged the normal way (via @{pr.repository.project_id.github_prefix}),"Comment when a forwardport PR gets closed then reopened, documents that the PR is now in a detached state.
-
-pr: the pr in question"
 runbot_merge.forwardport.ci.failed,{pr.ping}{ci} failed on this forward-port PR,"Comment when CI fails on a forward-port PR (which thus won't port any further, for now).
 
 pr: the pr in question
@@ -128,7 +125,7 @@ linked: the linked PR with a different next target
 next: next target for the current pr
 other: next target for the other pr"
 runbot_merge.forwardport.failure.conflict,"{pr.ping}the next pull request ({new.display_name}) is in conflict. You can merge the chain up to here by saying
-> @{pr.repository.project_id.fp_github_name} r+
+> @{pr.repository.project_id.github_prefix} r+
 {footer}","Comment when a forward port was created but is in conflict, warns of that & gives instructions for current PR.
 
 pr: the pr which was just forward ported
@@ -163,7 +160,7 @@ footer: some footer text"
 runbot_merge.forwardport.final,"{pr.ping}this PR targets {pr.target.name} and is the last of the forward-port chain{containing}
 {ancestors}
 To merge the full chain, use
-> @{pr.repository.project_id.fp_github_name} r+
+> @{pr.repository.project_id.github_prefix} r+
 {footer}","Comment when a forward port was created and is the last of a sequence (target the limit branch).
 
 pr: the new forward port
diff --git a/runbot_merge/migrations/15.0.1.10/pre-migration.py b/runbot_merge/migrations/15.0.1.10/pre-migration.py
new file mode 100644
index 00000000..a9ca29d3
--- /dev/null
+++ b/runbot_merge/migrations/15.0.1.10/pre-migration.py
@@ -0,0 +1,11 @@
+""" Migration for the unified commands parser, fp_github fields moved from
+forwardport to mergebot (one of them is removed but we might not care)
+"""
+def migrate(cr, version):
+    cr.execute("""
+    UPDATE ir_model_data
+       SET module = 'runbot_merge'
+     WHERE module = 'forwardport'
+       AND model = 'ir.model.fields'
+       AND name in ('fp_github_token', 'fp_github_name')
+    """)
diff --git a/runbot_merge/migrations/15.0.1.11/pre-migration.py b/runbot_merge/migrations/15.0.1.11/pre-migration.py
new file mode 100644
index 00000000..521e148b
--- /dev/null
+++ b/runbot_merge/migrations/15.0.1.11/pre-migration.py
@@ -0,0 +1,124 @@
+def move_fields(cr, *names):
+    cr.execute("""
+    UPDATE ir_model_data
+       SET module = 'runbot_merge'
+     WHERE module = 'forwardport'
+       AND model = 'runbot_merge_pull_requests'
+       AND name IN %s
+    """, [names])
+
+def migrate(cr, version):
+    # cleanup some old crap
+    cr.execute("""
+    ALTER TABLE runbot_merge_project_freeze
+        DROP COLUMN IF EXISTS release_label,
+        DROP COLUMN IF EXISTS bump_label
+    """)
+
+    # fw constraint moved to mergebot, alongside all the fields it constrains
+    cr.execute("""
+    UPDATE ir_model_data
+       SET module = 'runbot_merge'
+     WHERE module = 'forwardport'
+       AND model = 'ir.model.constraint'
+       AND name = 'constraint_runbot_merge_pull_requests_fw_constraint'
+    """)
+    move_fields(
+        cr, 'merge_date', 'refname',
+        'limit_id', 'source_id', 'parent_id', 'root_id', 'forwardport_ids',
+        'detach_reason', 'fw_policy')
+
+    # view depends on pr.state, which prevents changing the state column's type
+    # we can just drop the view and it'll be recreated by the db update
+    cr.execute("DROP VIEW runbot_merge_freeze_labels")
+    # convert a few data types
+    cr.execute("""
+    CREATE TYPE runbot_merge_pull_requests_priority_type
+        AS ENUM ('default', 'priority', 'alone');
+
+    CREATE TYPE runbot_merge_pull_requests_state_type
+        AS ENUM ('opened', 'closed', 'validated', 'approved', 'ready', 'merged', 'error');
+
+    CREATE TYPE runbot_merge_pull_requests_merge_method_type
+        AS ENUM ('merge', 'rebase-merge', 'rebase-ff', 'squash');
+
+    CREATE TYPE runbot_merge_pull_requests_status_type
+        AS ENUM ('pending', 'failure', 'success');
+
+
+    ALTER TABLE runbot_merge_pull_requests
+        ALTER COLUMN priority
+            TYPE runbot_merge_pull_requests_priority_type
+            USING CASE WHEN priority = 0
+                    THEN 'alone'
+                    ELSE 'default'
+                  END::runbot_merge_pull_requests_priority_type,
+        ALTER COLUMN state
+            TYPE runbot_merge_pull_requests_state_type
+            USING state::runbot_merge_pull_requests_state_type,
+        ALTER COLUMN merge_method
+            TYPE runbot_merge_pull_requests_merge_method_type
+            USING merge_method::runbot_merge_pull_requests_merge_method_type;
+    """)
+
+    cr.execute("""
+    ALTER TABLE runbot_merge_pull_requests
+     ADD COLUMN closed boolean not null default 'false',
+     ADD COLUMN error boolean not null default 'false',
+     ADD COLUMN skipchecks boolean not null default 'false',
+     ADD COLUMN cancel_staging boolean not null default 'false',
+
+     ADD COLUMN statuses text not null default '{}',
+     ADD COLUMN statuses_full text not null default '{}',
+     ADD COLUMN status runbot_merge_pull_requests_status_type not null default 'pending'
+    """)
+    # first pass: update all the new unconditional (or simple) fields
+    cr.execute("""
+    UPDATE runbot_merge_pull_requests p
+        SET closed = state = 'closed',
+            error = state = 'error',
+            skipchecks = priority = 'alone',
+            cancel_staging = priority = 'alone',
+            fw_policy = CASE fw_policy WHEN 'ci' THEN 'default' ELSE fw_policy END,
+            reviewed_by = CASE state
+                -- old version did not reset reviewer on PR update
+                WHEN 'opened' THEN NULL
+                WHEN 'validated' THEN NULL
+                -- if a PR predates the reviewed_by field, assign odoobot as reviewer
+                WHEN 'merged' THEN coalesce(reviewed_by, 2)
+                ELSE reviewed_by
+            END,
+            status = CASE state
+                WHEN 'validated' THEN 'success'
+                WHEN 'ready' THEN 'success'
+                WHEN 'merged' THEN 'success'
+                ELSE 'pending'
+            END::runbot_merge_pull_requests_status_type
+    """)
+
+    # the rest only gets updated if we have a matching commit which is not
+    # always the case
+    cr.execute("""
+    CREATE TEMPORARY TABLE parents ( id INTEGER not null, overrides jsonb not null );
+    WITH RECURSIVE parent_chain AS (
+         SELECT id, overrides::jsonb
+           FROM runbot_merge_pull_requests
+          WHERE parent_id IS NULL
+    UNION ALL
+         SELECT p.id, coalesce(pc.overrides || p.overrides::jsonb, pc.overrides, p.overrides::jsonb) as overrides
+           FROM runbot_merge_pull_requests p
+           JOIN parent_chain pc ON p.parent_id = pc.id
+    )
+    INSERT INTO parents SELECT * FROM parent_chain;
+    CREATE INDEX ON parents (id);
+
+    UPDATE runbot_merge_pull_requests p
+       SET statuses = jsonb_pretty(c.statuses::jsonb)::text,
+           statuses_full = jsonb_pretty(
+               c.statuses::jsonb
+            || coalesce((select overrides from parents where id = p.parent_id), '{}')
+            || overrides::jsonb
+           )::text
+      FROM runbot_merge_commit c
+     WHERE p.head = c.sha
+    """)
diff --git a/runbot_merge/models/commands.py b/runbot_merge/models/commands.py
index aa597bb8..7d097d47 100644
--- a/runbot_merge/models/commands.py
+++ b/runbot_merge/models/commands.py
@@ -1,7 +1,9 @@
 import enum
-from collections import abc
+from collections.abc import Iterator
 from dataclasses import dataclass, field
-from typing import Iterator, List, Optional, Union
+from functools import partial
+from operator import contains
+from typing import Callable, List, Optional, Union
 
 
 def tokenize(line: str) -> Iterator[str]:
@@ -21,6 +23,7 @@ def tokenize(line: str) -> Iterator[str]:
     if cur:
         yield cur
 
+
 def normalize(it: Iterator[str]) -> Iterator[str]:
     """Converts shorthand tokens to expanded version
     """
@@ -31,14 +34,12 @@ def normalize(it: Iterator[str]) -> Iterator[str]:
             case 'r-':
                 yield 'review'
                 yield '-'
-            case 'p':
-                yield 'priority'
             case _:
                 yield t
 
 
 @dataclass
-class Peekable(abc.Iterator[str]):
+class Peekable(Iterator[str]):
     it: Iterator[str]
     memo: Optional[str] = None
 
@@ -57,22 +58,23 @@ class Peekable(abc.Iterator[str]):
         return self.memo
 
 
-def assert_next(it: Iterator[str], val: str):
-    if (actual := next(it, None)) != val:
-        raise CommandError(f"expected {val!r}, got {actual!r}")
-
-
 class CommandError(Exception):
     pass
 
 
 class Approve:
-    def __str__(self):
+    def __init__(self, ids: Optional[List[int]] = None) -> None:
+        self.ids = ids
+
+    def __str__(self) -> str:
+        if self.ids is not None:
+            ids = ','.join(map(str, self.ids))
+            return f"r={ids}"
         return 'review+'
 
 
 class Reject:
-    def __str__(self):
+    def __str__(self) -> str:
         return 'review-'
 
 
@@ -82,17 +84,17 @@ class MergeMethod(enum.Enum):
     REBASE_MERGE = 'rebase-merge'
     MERGE = 'merge'
 
-    def __str__(self):
+    def __str__(self) -> str:
         return self.value
 
 
 class Retry:
-    def __str__(self):
+    def __str__(self) -> str:
         return 'retry'
 
 
 class Check:
-    def __str__(self):
+    def __str__(self) -> str:
         return 'check'
 
 
@@ -100,7 +102,7 @@ class Check:
 class Override:
     statuses: List[str] = field(default_factory=list)
 
-    def __str__(self):
+    def __str__(self) -> str:
         return f"override={','.join(self.statuses)}"
 
 
@@ -108,124 +110,199 @@ class Override:
 class Delegate:
     users: List[str] = field(default_factory=list)
 
-    def __str__(self):
+    def __str__(self) -> str:
         if not self.users:
             return 'delegate+'
         return f"delegate={','.join(self.users)}"
 
 
-class Priority(enum.IntEnum):
-    NUKE = 0
-    PRIORITIZE = 1
-    DEFAULT = 2
+class Priority(enum.Enum):
+    DEFAULT = enum.auto()
+    PRIORITY = enum.auto()
+    ALONE = enum.auto()
+
+    def __str__(self) -> str:
+        return self.name.lower()
 
 
-Command = Union[Approve, Reject, MergeMethod, Retry, Check, Override, Delegate, Priority]
+class CancelStaging:
+    def __str__(self) -> str:
+        return "cancel=staging"
 
 
-def parse_mergebot(line: str) -> Iterator[Command]:
-    it = Peekable(normalize(tokenize(line)))
-    for token in it:
-        match token:
-            case 'review':
-                match next(it):
-                    case '+':
-                        yield Approve()
-                    case '-':
-                        yield Reject()
-                    case t:
-                        raise CommandError(f"unknown review {t!r}")
-            case 'squash':
-                yield MergeMethod.SQUASH
-            case 'rebase-ff':
-                yield MergeMethod.REBASE_FF
-            case 'rebase-merge':
-                yield MergeMethod.REBASE_MERGE
-            case 'merge':
-                yield MergeMethod.MERGE
-            case 'retry':
-                yield Retry()
-            case 'check':
-                yield Check()
-            case 'override':
-                assert_next(it, '=')
-                ci = [next(it)]
-                while it.peek() == ',':
-                    next(it)
-                    ci.append(next(it))
-                yield Override(ci)
-            case 'delegate':
-                match next(it):
-                    case '+':
-                        yield Delegate()
-                    case '=':
-                        delegates = [next(it).lstrip('#@')]
-                        while it.peek() == ',':
-                            next(it)
-                            delegates.append(next(it).lstrip('#@'))
-                        yield Delegate(delegates)
-                    case d:
-                        raise CommandError(f"unknown delegation {d!r}")
-            case 'priority':
-                assert_next(it, '=')
-                yield Priority(int(next(it)))
-            case c:
-                raise CommandError(f"unknown command {c!r}")
+class SkipChecks:
+    def __str__(self) -> str:
+        return 'skipchecks'
 
 
-class FWApprove:
-    def __str__(self):
-        return 'review+'
+class FW(enum.Enum):
+    DEFAULT = enum.auto()
+    SKIPCI = enum.auto()
+    SKIPMERGE = enum.auto()
 
-
-@dataclass
-class CI:
-    run: bool
-    def __str__(self):
-        return 'ci' if self.run else 'skipci'
+    def __str__(self) -> str:
+        return f'fw={self.name.lower()}'
 
 
 @dataclass
 class Limit:
     branch: Optional[str]
 
-    def __str__(self):
+    def __str__(self) -> str:
         if self.branch is None:
             return 'ignore'
         return f'up to {self.branch}'
 
 
 class Close:
-    def __str__(self):
+    def __str__(self) -> str:
         return 'close'
 
 
-FWCommand = Union[FWApprove, CI, Limit, Close]
+Command = Union[
+    Approve,
+    CancelStaging,
+    Close,
+    Check,
+    Delegate,
+    FW,
+    Limit,
+    MergeMethod,
+    Override,
+    Priority,
+    Reject,
+    Retry,
+    SkipChecks,
+]
 
 
-def parse_forwardbot(line: str) -> Iterator[FWCommand]:
-    it = Peekable(normalize(tokenize(line)))
-    for token in it:
-        match token:
-            case 'review':
-                match next(it):
-                    case '+':
-                        yield FWApprove()
-                    case t:
-                        raise CommandError(f"unknown review {t!r}", True)
-            case 'ci':
-                yield CI(True)
-            case 'skipci':
-                yield CI(False)
-            case 'ignore':
-                yield Limit(None)
-            case 'up':
-                assert_next(it, 'to')
-                if limit := next(it, None):
-                    yield Limit(limit)
+class Parser:
+    def __init__(self, line: str) -> None:
+        self.it = Peekable(normalize(tokenize(line)))
+
+    def __iter__(self) -> Iterator[Command]:
+        for token in self.it:
+            if token.startswith("NOW"):
+                # any number of ! is allowed
+                if token.startswith("NOW!"):
+                    yield Priority.ALONE
+                elif token == "NOW":
+                    yield Priority.PRIORITY
                 else:
-                    raise CommandError("please provide a branch to forward-port to.", True)
-            case 'close':
-                yield Close()
-            case c:
-                raise CommandError(f"unknown command {c!r}", True)
+                    raise CommandError(f"unknown command {token!r}")
+                yield SkipChecks()
+                yield CancelStaging()
+                continue
+
+            handler = getattr(type(self), f'parse_{token.replace("-", "_")}', None)
+            if handler:
+                yield handler(self)
+            elif '!' in token:
+                raise CommandError("skill issue, noob")
+            else:
+                raise CommandError(f"unknown command {token!r}")
+
+    def assert_next(self, val: str) -> None:
+        if (actual := next(self.it, None)) != val:
+            raise CommandError(f"expected {val!r}, got {actual!r}")
+
+    def check_next(self, val: str) -> bool:
+        if self.it.peek() == val:
+            self.it.memo = None # consume peeked value
+            return True
+        return False
+
+    def parse_review(self) -> Union[Approve, Reject]:
+        t = next(self.it, None)
+        if t == '+':
+            return Approve()
+        if t == '-':
+            return Reject()
+        if t == '=':
+            t = next(self.it, None)
+            if not (t and t.isdecimal()):
+                raise CommandError(f"expected PR ID to approve, found {t!r}")
+
+            ids = [int(t)]
+            while self.check_next(','):
+                id = next(self.it, None)
+                if id and id.isdecimal():
+                    ids.append(int(id))
+                else:
+                    raise CommandError(f"expected PR ID to approve, found {id!r}")
+            return Approve(ids)
+
+        raise CommandError(f"unknown review {t!r}")
+
+    def parse_squash(self) -> MergeMethod:
+        return MergeMethod.SQUASH
+
+    def parse_rebase_ff(self) -> MergeMethod:
+        return MergeMethod.REBASE_FF
+
+    def parse_rebase_merge(self) -> MergeMethod:
+        return MergeMethod.REBASE_MERGE
+
+    def parse_merge(self) -> MergeMethod:
+        return MergeMethod.MERGE
+
+    def parse_retry(self) -> Retry:
+        return Retry()
+
+    def parse_check(self) -> Check:
+        return Check()
+
+    def parse_override(self) -> Override:
+        self.assert_next('=')
+        ci = [next(self.it)]
+        while self.check_next(','):
+            ci.append(next(self.it))
+        return Override(ci)
+
+    def parse_delegate(self) -> Delegate:
+        match next(self.it, None):
+            case '+':
+                return Delegate()
+            case '=':
+                delegates = [next(self.it).lstrip('#@')]
+                while self.check_next(','):
+                    delegates.append(next(self.it).lstrip('#@'))
+                return Delegate(delegates)
+            case d:
+                raise CommandError(f"unknown delegation {d!r}")
+
+    def parse_default(self) -> Priority:
+        return Priority.DEFAULT
+
+    def parse_priority(self) -> Priority:
+        return Priority.PRIORITY
+
+    def parse_alone(self) -> Priority:
+        return Priority.ALONE
+
+    def parse_cancel(self) -> CancelStaging:
+        return CancelStaging()
+
+    def parse_skipchecks(self) -> SkipChecks:
+        return SkipChecks()
+
+    def parse_fw(self) -> FW:
+        self.assert_next('=')
+        f = next(self.it, "")
+        try:
+            return FW[f.upper()]
+        except KeyError:
+            raise CommandError(f"unknown fw configuration {f or None!r}") from None
+
+    def parse_ignore(self) -> Limit:
+        return Limit(None)
+
+    def parse_up(self) -> Limit:
+        self.assert_next('to')
+        if limit := next(self.it, None):
+            return Limit(limit)
+        else:
+            raise CommandError("please provide a branch to forward-port to.")
+
+    def parse_close(self) -> Close:
+        return Close()
diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index 9e335267..6f88e117 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -37,11 +37,10 @@ class Project(models.Model):
         required=True,
         default="hanson", # mergebot du bot du bot du~
         help="Prefix (~bot name) used when sending commands from PR "
-             "comments e.g. [hanson retry] or [hanson r+ p=1]",
+             "comments e.g. [hanson retry] or [hanson r+ priority]",
     )
     fp_github_token = fields.Char()
     fp_github_name = fields.Char(store=True, compute="_compute_git_identity")
-    fp_github_email = fields.Char(store=True, compute="_compute_git_identity")
 
     batch_limit = fields.Integer(
         default=8, group_operator=None, help="Maximum number of PRs staged together")
@@ -105,7 +104,7 @@ class Project(models.Model):
     def _compute_git_identity(self):
         s = requests.Session()
         for project in self:
-            if not project.fp_github_token or (project.fp_github_name and project.fp_github_email):
+            if project.fp_github_name or not project.fp_github_token:
                 continue
 
             r0 = s.get('https://api.github.com/user', headers={
@@ -117,25 +116,6 @@ class Project(models.Model):
 
             user = r0.json()
             project.fp_github_name = user['name'] or user['login']
-            if email := user['email']:
-                project.fp_github_email = email
-                continue
-
-            if 'user:email' not in set(re.split(r',\s*', r0.headers['x-oauth-scopes'])):
-                raise UserError("The forward-port github token needs the user:email scope to fetch the bot's identity.")
-            r1 = s.get('https://api.github.com/user/emails', headers={
-                'Authorization': 'token %s' % project.fp_github_token
-            })
-            if not r1.ok:
-                _logger.error("Failed to fetch forward bot emails for project %s: %s", project.name, r1.text or r1.content)
-                continue
-            project.fp_github_email = next((
-                entry['email']
-                for entry in r1.json()
-                if entry['primary']
-            ), None)
-            if not project.fp_github_email:
-                raise UserError("The forward-port bot needs a public or primary email set up.")
 
     def _check_stagings(self, commit=False):
         # check branches with an active staging
@@ -173,18 +153,17 @@ class Project(models.Model):
                 if commit:
                     self.env.cr.commit()
 
-    def _find_commands(self, comment: str) -> List[Tuple[str, str]]:
+    def _find_commands(self, comment: str) -> List[str]:
         """Tries to find all the lines starting (ignoring leading whitespace)
         with either the merge or the forward port bot identifiers.
 
         For convenience, the identifier *can* be prefixed with an ``@`` or
         ``#``, and suffixed with a ``:``.
         """
-        logins = '|'.join(map(re.escape, filter(None, [self.github_prefix, self.fp_github_name])))
         # horizontal whitespace (\s - {\n, \r}), but Python doesn't have \h or \p{Blank}
         h = r'[^\S\r\n]'
         return re.findall(
-            fr'^{h}*[@|#]?({logins})(?:{h}+|:{h}*)(.*)$',
+            fr'^{h}*[@|#]?{self.github_prefix}(?:{h}+|:{h}*)(.*)$',
             comment, re.MULTILINE | re.IGNORECASE)
 
     def _has_branch(self, name):
diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py
index 000b8835..34758b7f 100644
--- a/runbot_merge/models/project_freeze/__init__.py
+++ b/runbot_merge/models/project_freeze/__init__.py
@@ -339,7 +339,10 @@ class FreezeWizard(models.Model):
             )
 
         all_prs = self.release_pr_ids.pr_id | self.bump_pr_ids.pr_id
-        all_prs.state = 'merged'
+        all_prs.write({
+            'merge_date': fields.Datetime.now(),
+            'reviewed_by': self.env.user.partner_id.id,
+        })
         self.env['runbot_merge.pull_requests.feedback'].create([{
             'repository': pr.repository.id,
             'pull_request': pr.number,
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 6e4d63db..158cff5a 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import ast
 import collections
 import contextlib
@@ -5,10 +7,10 @@ import datetime
 import itertools
 import json
 import logging
-import pprint
 import re
 import time
-from typing import Optional, Union, List, Literal
+from functools import reduce
+from typing import Optional, Union, List, Iterator, Tuple
 
 import sentry_sdk
 import werkzeug
@@ -16,6 +18,7 @@ import werkzeug
 from odoo import api, fields, models, tools
 from odoo.exceptions import ValidationError
 from odoo.osv import expression
+from odoo.tools import html_escape
 from . import commands
 
 from .. import github, exceptions, controllers, utils
@@ -219,7 +222,7 @@ All substitutions are tentatively applied sequentially to the input.
             # don't go through controller because try_closing does weird things
             # for safety / race condition reasons which ends up committing
             # and breaks everything
-            pr_id.state = 'closed'
+            pr_id.closed = True
 
         self.env.ref('runbot_merge.pr.load.fetched')._send(
             repository=self,
@@ -305,18 +308,37 @@ class Branch(models.Model):
 
 
 ACL = collections.namedtuple('ACL', 'is_admin is_reviewer is_author')
+def enum(model: str, field: str) -> Tuple[str, str]:
+    n = f'{model.replace(".", "_")}_{field}_type'
+    return n, n
 class PullRequests(models.Model):
-    _name = _description = 'runbot_merge.pull_requests'
+    _name = 'runbot_merge.pull_requests'
+    _description = "Pull Request"
+    _inherit = ['mail.thread']
     _order = 'number desc'
     _rec_name = 'number'
 
     id: int
     display_name: str
 
-    target = fields.Many2one('runbot_merge.branch', required=True, index=True)
+    target = fields.Many2one('runbot_merge.branch', required=True, index=True, tracking=True)
     repository = fields.Many2one('runbot_merge.repository', required=True)
     # NB: check that target & repo have same project & provide project related?
 
+    closed = fields.Boolean(default=False, tracking=True)
+    error = fields.Boolean(string="in error", default=False, tracking=True)
+    skipchecks = fields.Boolean(
+        string="Skips Checks",
+        default=False, tracking=True,
+        help="Forces entire batch to be ready, skips validation and approval",
+    )
+    cancel_staging = fields.Boolean(
+        string="Cancels Stagings",
+        default=False, tracking=True,
+        help="Cancels current staging on target branch when becoming ready"
+    )
+    merge_date = fields.Datetime(tracking=True)
+
     state = fields.Selection([
         ('opened', 'Opened'),
         ('closed', 'Closed'),
@@ -326,41 +348,56 @@ class PullRequests(models.Model):
         # staged?
         ('merged', 'Merged'),
         ('error', 'Error'),
-    ], default='opened', index=True)
+    ],
+        compute='_compute_state', store=True, default='opened',
+        index=True, tracking=True, column_type=enum(_name, 'state'),
+    )
 
     number = fields.Integer(required=True, index=True, group_operator=None)
     author = fields.Many2one('res.partner', index=True)
-    head = fields.Char(required=True)
+    head = fields.Char(required=True, tracking=True)
     label = fields.Char(
         required=True, index=True,
         help="Label of the source branch (owner:branchname), used for "
              "cross-repository branch-matching"
     )
+    refname = fields.Char(compute='_compute_refname')
     message = fields.Text(required=True)
-    draft = fields.Boolean(default=False, required=True)
-    squash = fields.Boolean(default=False)
+    draft = fields.Boolean(
+        default=False, required=True, tracking=True,
+        help="A draft PR can not be merged",
+    )
+    squash = fields.Boolean(default=False, tracking=True)
     merge_method = fields.Selection([
         ('merge', "merge directly, using the PR as merge commit message"),
         ('rebase-merge', "rebase and merge, using the PR as merge commit message"),
         ('rebase-ff', "rebase and fast-forward"),
         ('squash', "squash"),
-    ], default=False)
+    ], default=False, tracking=True, column_type=enum(_name, 'merge_method'))
     method_warned = fields.Boolean(default=False)
 
-    reviewed_by = fields.Many2one('res.partner', index=True)
+    reviewed_by = fields.Many2one('res.partner', index=True, tracking=True)
     delegates = fields.Many2many('res.partner', help="Delegate reviewers, not intrinsically reviewers but can review this PR")
-    priority = fields.Integer(default=2, index=True, group_operator=None)
-
-    overrides = fields.Char(required=True, default='{}')
-    statuses = fields.Text(
-        compute='_compute_statuses',
-        help="Copy of the statuses from the HEAD commit, as a Python literal"
+    priority = fields.Selection([
+        ('default', "Default"),
+        ('priority', "Priority"),
+        ('alone', "Alone"),
+    ], default='default', index=True, group_operator=None, required=True,
+        column_type=enum(_name, 'priority'),
     )
+
+    overrides = fields.Char(required=True, default='{}', tracking=True)
+    statuses = fields.Text(help="Copy of the statuses from the HEAD commit, as a Python literal", default="{}")
     statuses_full = fields.Text(
         compute='_compute_statuses',
-        help="Compilation of the full status of the PR (commit statuses + overrides), as JSON"
+        help="Compilation of the full status of the PR (commit statuses + overrides), as JSON",
+        store=True,
     )
-    status = fields.Char(compute='_compute_statuses')
+    status = fields.Selection([
+        ('pending', 'Pending'),
+        ('failure', 'Failure'),
+        ('success', 'Success'),
+    ], compute='_compute_statuses', store=True, column_type=enum(_name, 'status'))
     previous_failure = fields.Char(default='{}')
 
     batch_id = fields.Many2one('runbot_merge.batch', string="Active Batch", compute='_compute_active_batch', store=True)
@@ -384,15 +421,47 @@ class PullRequests(models.Model):
     repo_name = fields.Char(related='repository.name')
     message_title = fields.Char(compute='_compute_message_title')
 
-    ping = fields.Char(compute='_compute_ping')
+    ping = fields.Char(compute='_compute_ping', recursive=True)
 
-    @api.depends('author.github_login', 'reviewed_by.github_login')
+    fw_policy = fields.Selection([
+        ('default', "Default"),
+        ('skipci', "Skip CI"),
+    ], required=True, default="default", string="Forward Port Policy")
+    source_id = fields.Many2one('runbot_merge.pull_requests', index=True, help="the original source of this FP even if parents were detached along the way")
+    parent_id = fields.Many2one(
+        'runbot_merge.pull_requests', index=True,
+        help="a PR with a parent is an automatic forward port",
+        tracking=True,
+    )
+    root_id = fields.Many2one('runbot_merge.pull_requests', compute='_compute_root', recursive=True)
+    forwardport_ids = fields.One2many('runbot_merge.pull_requests', 'source_id')
+    limit_id = fields.Many2one('runbot_merge.branch', help="Up to which branch should this PR be forward-ported", tracking=True)
+
+    detach_reason = fields.Char()
+
+    _sql_constraints = [(
+        'fw_constraint',
+        'check(source_id is null or num_nonnulls(parent_id, detach_reason) = 1)',
+        "fw PRs must either be attached or have a reason for being detached",
+    )]
+
+    @api.depends('label')
+    def _compute_refname(self):
+        for pr in self:
+            pr.refname = pr.label.split(':', 1)[-1]
+
+    @api.depends(
+        'author.github_login', 'reviewed_by.github_login',
+        'source_id.author.github_login', 'source_id.reviewed_by.github_login',
+    )
     def _compute_ping(self):
         for pr in self:
-            s = ' '.join(
-                f'@{p.github_login}'
-                for p in (pr.author | pr.reviewed_by )
-            )
+            if source := pr.source_id:
+                contacts = source.author | source.reviewed_by | pr.reviewed_by
+            else:
+                contacts = pr.author | pr.reviewed_by
+
+            s = ' '.join(f'@{p.github_login}' for p in contacts)
             pr.ping = s and (s + ' ')
 
     @api.depends('repository.name', 'number')
@@ -404,6 +473,11 @@ class PullRequests(models.Model):
             pr.url = str(base.join(path))
             pr.github_url = str(gh_base.join(path))
 
+    @api.depends('parent_id.root_id')
+    def _compute_root(self):
+        for p in self:
+            p.root_id = reduce(lambda _, p: p, self._iter_ancestors())
+
     @api.depends('message')
     def _compute_message_title(self):
         for pr in self:
@@ -411,7 +485,7 @@ class PullRequests(models.Model):
 
     @api.depends('repository.name', 'number', 'message')
     def _compute_display_name(self):
-        return super(PullRequests, self)._compute_display_name()
+        return super()._compute_display_name()
 
     def name_get(self):
         name_template = '%(repo_name)s#%(number)d'
@@ -438,10 +512,7 @@ class PullRequests(models.Model):
 
     @property
     def _approved(self):
-        return self.state in ('approved', 'ready') or any(
-            p.priority == 0
-            for p in (self | self._linked_prs)
-        )
+        return self.state in ('approved', 'ready')
 
     @property
     def _ready(self):
@@ -462,72 +533,39 @@ class PullRequests(models.Model):
         ]) - self
 
     # missing link to other PRs
-    @api.depends('priority', 'state', 'squash', 'merge_method', 'batch_id.active', 'label')
+    @api.depends('state')
     def _compute_is_blocked(self):
         self.blocked = False
+        requirements = (
+            lambda p: not p.draft,
+            lambda p: p.squash or p.merge_method,
+            lambda p: p.state == 'ready' \
+                  or any(pr.skipchecks for pr in (p | p._linked_prs)) \
+                 and all(pr.state != 'error' for pr in (p | p._linked_prs))
+        )
+        messages = ('is in draft', 'has no merge method', 'is not ready')
         for pr in self:
             if pr.state in ('merged', 'closed'):
                 continue
 
-            linked = pr._linked_prs
-            # check if PRs are configured (single commit or merge method set)
-            if not (pr.squash or pr.merge_method):
-                pr.blocked = 'has no merge method'
-                continue
-            other_unset = next((p for p in linked if not (p.squash or p.merge_method)), None)
-            if other_unset:
-                pr.blocked = "linked PR %s has no merge method" % other_unset.display_name
-                continue
+            blocking, message = next((
+                (blocking, message)
+                for blocking in (pr | pr._linked_prs)
+                for requirement, message in zip(requirements, messages)
+                if not requirement(blocking)
+            ), (None, None))
+            if blocking == pr:
+                pr.blocked = message
+            elif blocking:
+                pr.blocked = f"linked PR {blocking.display_name} {message}"
 
-            # check if any PR in the batch is p=0 and none is in error
-            if any(p.priority == 0 for p in (pr | linked)):
-                if pr.state == 'error':
-                    pr.blocked = "in error"
-                other_error = next((p for p in linked if p.state == 'error'), None)
-                if other_error:
-                    pr.blocked = "linked pr %s in error" % other_error.display_name
-                # if none is in error then none is blocked because p=0
-                # "unblocks" the entire batch
-                continue
-
-            if pr.state != 'ready':
-                pr.blocked = 'not ready'
-                continue
-
-            unready = next((p for p in linked if p.state != 'ready'), None)
-            if unready:
-                pr.blocked = 'linked pr %s is not ready' % unready.display_name
-                continue
-
-    def _get_overrides(self):
+    def _get_overrides(self) -> dict[str, dict[str, str]]:
+        if self.parent_id:
+            return self.parent_id._get_overrides() | json.loads(self.overrides)
         if self:
             return json.loads(self.overrides)
         return {}
 
-    @api.depends('head', 'repository.status_ids', 'overrides')
-    def _compute_statuses(self):
-        Commits = self.env['runbot_merge.commit']
-        for pr in self:
-            c = Commits.search([('sha', '=', pr.head)])
-            st = json.loads(c.statuses or '{}')
-            statuses = {**st, **pr._get_overrides()}
-            pr.statuses_full = json.dumps(statuses)
-            if not statuses:
-                pr.status = pr.statuses = False
-                continue
-
-            pr.statuses = pprint.pformat(st)
-
-            st = 'success'
-            for ci in pr.repository.status_ids._for_pr(pr):
-                v = (statuses.get(ci.context) or {'state': 'pending'})['state']
-                if v in ('error', 'failure'):
-                    st = 'failure'
-                    break
-                if v == 'pending':
-                    st = 'pending'
-            pr.status = st
-
     @api.depends('batch_ids.active')
     def _compute_active_batch(self):
         for r in self:
@@ -562,6 +600,16 @@ class PullRequests(models.Model):
             'closing': closing,
         })
 
+    def _iter_ancestors(self) -> Iterator[PullRequests]:
+        while self:
+            yield self
+            self = self.parent_id
+
+    def _iter_descendants(self) -> Iterator[PullRequests]:
+        pr = self
+        while pr := self.search([('parent_id', '=', pr.id)]):
+            yield pr
+
     def _parse_commands(self, author, comment, login):
         assert self, "parsing commands must be executed in an actual PR"
 
@@ -574,34 +622,34 @@ class PullRequests(models.Model):
             )
             return 'ok'
 
-        def feedback(message: Optional[str] = None, close: bool = False, token: Literal['github_token', 'fp_github_token'] = 'github_token'):
+        def feedback(message: Optional[str] = None, close: bool = False):
             self.env['runbot_merge.pull_requests.feedback'].create({
                 'repository': self.repository.id,
                 'pull_request': self.number,
                 'message': message,
                 'close': close,
-                'token_field': token,
             })
         try:
-            cmds: List[Union[commands.Command, commands.FWCommand]] = [
+            cmds: List[commands.Command] = [
                 ps
-                for bot, line in commandlines
-                for ps in (commands.parse_mergebot(line) if bot.casefold() == self.repository.project_id.github_prefix.casefold() else commands.parse_forwardbot(line))
+                for line in commandlines
+                for ps in commands.Parser(line)
             ]
         except Exception as e:
             _logger.info(
                 "error %s while parsing comment of %s (%s): %s",
                 e,
-                author.github_login, author.display_name,
+                login, name,
                 utils.shorten(comment['body'] or '', 50),
                 exc_info=True
             )
-            feedback(message=f"@{login} {e.args[0]}", token='fp_github_token' if len(e.args) >= 2 and e.args[1] else 'github_token')
+            feedback(message=f"@{login} {e.args[0]}")
             return 'error'
 
         is_admin, is_reviewer, is_author = self._pr_acl(author)
+        _source_admin, source_reviewer, source_author = self.source_id._pr_acl(author)
 
-        if not (is_author or any(isinstance(cmd, commands.Override) for cmd in cmds)):
+        if not (is_author or self.source_id or (any(isinstance(cmd, commands.Override) for cmd in cmds) and author.override_rights)):
             # no point even parsing commands
             _logger.info("ignoring comment of %s (%s): no ACL to %s",
                           login, name, self.display_name)
@@ -614,50 +662,75 @@ class PullRequests(models.Model):
 
         rejections = []
         for command in cmds:
-            fwbot, msg = False, None
+            msg = None
             match command:
                 case commands.Approve() if self.draft:
                     msg = "draft PRs can not be approved."
-                case commands.Approve() if is_reviewer:
-                    oldstate = self.state
-                    newstate = RPLUS.get(self.state)
-                    if not author.email:
-                        msg = "I must know your email before you can review PRs. Please contact an administrator."
-                    elif not newstate:
-                        msg = "this PR is already reviewed, reviewing it again is useless."
+                case commands.Approve() if self.parent_id:
+                    # rules are a touch different for forwardport PRs:
+                    valid = lambda _: True if command.ids is None else lambda n: n in command.ids
+                    _, source_reviewer, source_author = self.source_id._pr_acl(author)
+
+                    ancestors = list(self._iter_ancestors())
+                    # - reviewers on the original can approve any forward port
+                    if source_reviewer:
+                        approveable = ancestors
                     else:
-                        self.state = newstate
-                        self.reviewed_by = author
-                    _logger.debug(
-                        "r+ on %s by %s (%s->%s) status=%s message? %s",
-                        self.display_name, author.github_login,
-                        oldstate, newstate or oldstate,
-                        self.status, self.status == 'failure'
-                    )
-                    if self.status == 'failure':
-                        # the normal infrastructure is for failure and
-                        # prefixes messages with "I'm sorry"
-                        self.env.ref("runbot_merge.command.approve.failure")._send(
-                            repository=self.repository,
-                            pull_request=self.number,
-                            format_args={'user': login, 'pr': self},
-                        )
+                        # between the first merged ancestor and self
+                        mergeors = list(itertools.dropwhile(
+                            lambda p: p.state != 'merged',
+                            reversed(ancestors),
+                        ))
+                        # between the first ancestor the current user can review and self
+                        reviewors = list(itertools.dropwhile(
+                            lambda p: not p._pr_acl(author).is_reviewer,
+                            reversed(ancestors),
+                        ))
+
+                        # source author can approve any descendant of a merged
+                        # forward port (or source), people with review rights
+                        # to a forward port have review rights to its
+                        # descendants, if both apply use the most favorable
+                        # (largest number of PRs)
+                        if source_author and len(mergeors) > len(reviewors):
+                            approveable = mergeors
+                        else:
+                            approveable = reviewors
+
+                    if approveable:
+                        for pr in approveable:
+                            if not (pr.state in RPLUS and valid(pr.number)):
+                                continue
+                            msg = pr._approve(author, login)
+                            if msg:
+                                break
+                    else:
+                        msg = f"you can't {command} you silly little bean."
+                case commands.Approve() if is_reviewer:
+                    if command.ids is not None and command.ids != [self.number]:
+                        msg = f"tried to approve PRs {command.ids} but the current PR is {self.number}"
+                    else:
+                        msg = self._approve(author, login)
                 case commands.Reject() if is_author:
-                    newstate = RMINUS.get(self.state)
-                    if self.priority == 0 or newstate:
-                        if newstate:
-                            self.state = newstate
-                        if self.priority == 0:
-                            self.priority = 1
+                    batch = self | self._linked_prs
+                    if cancellers := batch.filtered('cancel_staging'):
+                        cancellers.cancel_staging = False
+                    if (skippers := batch.filtered('skipchecks')) or self.reviewed_by:
+                        if self.error:
+                            self.error = False
+                        if self.reviewed_by:
+                            self.reviewed_by = False
+                        if skippers:
+                            skippers.skipchecks = False
                             self.env.ref("runbot_merge.command.unapprove.p0")._send(
                                 repository=self.repository,
                                 pull_request=self.number,
-                                format_args={'user': login, 'pr': self},
+                                format_args={'user': login, 'pr': skippers[:1]},
                             )
                         self.unstage("unreviewed (r-) by %s", login)
                     else:
                         msg = "r- makes no sense in the current PR state."
-                case commands.MergeMethod() as command if is_reviewer:
+                case commands.MergeMethod() if is_reviewer:
                     self.merge_method = command.value
                     explanation = next(label for value, label in type(self).merge_method.selection if value == command.value)
                     self.env.ref("runbot_merge.command.method")._send(
@@ -665,9 +738,9 @@ class PullRequests(models.Model):
                         pull_request=self.number,
                         format_args={'new_method': explanation, 'pr': self, 'user': login},
                     )
-                case commands.Retry() if is_author:
-                    if self.state == 'error':
-                        self.state = 'ready'
+                case commands.Retry() if is_author or source_author:
+                    if self.error:
+                        self.error = False
                     else:
                         msg = "retry makes no sense when the PR is not in error."
                 case commands.Check() if is_author:
@@ -687,13 +760,23 @@ class PullRequests(models.Model):
                             })
                     delegates.write({'delegate_reviewer': [(4, self.id, 0)]})
                 case commands.Priority() if is_admin:
-                    self.priority = int(command)
-                    if command is commands.Priority.NUKE:
-                        self.target.active_staging_id.cancel(
-                            "P=0 on %s by %s, unstaging target %s",
-                            self.display_name,
-                            author.github_login, self.target.name,
-                        )
+                    self.priority = str(command)
+                case commands.SkipChecks() if is_admin:
+                    self.skipchecks = True
+                    self.reviewed_by = author
+                    for p in self.batch_id.prs - self:
+                        if not p.reviewed_by:
+                            p.reviewed_by = author
+                case commands.CancelStaging() if is_admin:
+                    self.cancel_staging = True
+                    # FIXME: remove this when skipchecks properly affects state,
+                    #        maybe: staging cancellation should then only occur
+                    #        when a cancel_staging PR transitions to ready, or
+                    #        a ready PR is flagged as cancelling staging
+                    self.target.active_staging_id.cancel(
+                        "Unstaged by %s on %s",
+                        author.github_login, self.display_name,
+                    )
                 case commands.Override(statuses):
                     for status in statuses:
                         overridable = author.override_rights\
@@ -716,68 +799,89 @@ class PullRequests(models.Model):
                         else:
                             msg = f"you are not allowed to override {status!r}."
                 # FW
-                case commands.FWApprove():
-                    fwbot = True
-                    if not self.source_id:
-                        msg = "I can only do this on forward-port PRs and this is not one, see {}.".format(
-                            self.repository.project_id.github_prefix
-                        )
-                    elif not self.parent_id:
-                        msg = "I can only do this on unmodified forward-port PRs, ask {}.".format(
-                            self.repository.project_id.github_prefix
-                        )
+                case commands.Close() if source_author:
+                    feedback(close=True)
+                case commands.FW():
+                    if source_reviewer or is_reviewer:
+                        (self.source_id or self).fw_policy = command.name.lower()
+                        match command:
+                            case commands.FW.DEFAULT:
+                                message = "Waiting for CI to create followup forward-ports."
+                            case commands.FW.SKIPCI:
+                                message = "Not waiting for CI to create followup forward-ports."
+                            case commands.FW.SKIPMERGE:
+                                message = "Not waiting for merge to create followup forward-ports."
+                        feedback(message=message)
                     else:
-                        merge_bot = self.repository.project_id.github_prefix
-                        # FIXME: classification of messages from child PR :(
-                        # don't update the root ever
-                        for pr in (p for p in self._iter_ancestors() if p.parent_id if p.state in RPLUS):
-                            # only the author is delegated explicitely on the
-                            pr._parse_commands(author, {**comment, 'body': merge_bot + ' r+'}, login)
-                case commands.Close() if self.source_id._pr_acl(author).is_reviewer:
-                    feedback(close=True, token='fp_github_token')
-                case commands.CI(run):
-                    pr = (self.source_id or self)
-                    if pr._pr_acl(author).is_reviewer:
-                        pr.fw_policy = 'ci' if run else 'skipci'
-                        feedback(
-                            message="Waiting for CI to create followup forward-ports." if run else "Not waiting for CI to create followup forward-ports.",
-                            token='fp_github_token',
-                        )
-                    else:
-                        fwbot = True
-                        msg = "you can't configure ci."
-                case commands.Limit(branch):
-                    fwbot = True
-                    if is_author:
-                        ping, msg = self._maybe_update_limit(branch or self.target.name)
-                        if not ping:
-                            feedback(message=msg, token='fp_github_token')
-                            msg = None
-                    else:
-                        msg = "you can't set a forward-port limit."
+                        msg = "you can't configure forward-port CI."
+                case commands.Limit(branch) if is_author:
+                    ping, msg = self._maybe_update_limit(branch or self.target.name)
+                    if not ping:
+                        feedback(message=msg)
+                        msg = None
+                case commands.Limit():
+                    msg = "you can't set a forward-port limit."
                 # NO!
                 case _:
                     msg = f"you can't {command}. Skill issue."
             if msg is not None:
-                rejections.append((fwbot, msg))
+                rejections.append(msg)
 
+        cmdstr = ', '.join(map(str, cmds))
         if not rejections:
-            _logger.info("%s (%s) applied %s", login, name, cmds)
-            return 'applied ' + ', '.join(map(str, cmds))
+            _logger.info("%s (%s) applied %s", login, name, cmdstr)
+            self.env.cr.precommit.data['change-author'] = author.id
+            return 'applied ' + cmdstr
 
         self.env.cr.rollback()
-        rejections_list = ''.join(f'\n- {r}' for fw, r in rejections if not fw)
-        fw_rejections_list = ''.join(f'\n- {r}' for fw, r in rejections if fw)
-        _logger.info("%s (%s) tried to apply %s%s", login, name, cmds, rejections_list + fw_rejections_list)
+        rejections_list = ''.join(f'\n- {r}' for r in rejections)
+        _logger.info("%s (%s) tried to apply %s%s", login, name, cmdstr, rejections_list)
         footer = '' if len(cmds) == len(rejections) else "\n\nFor your own safety I've ignored everything in your comment."
         if rejections_list:
             rejections = ' ' + rejections_list.removeprefix("\n- ") if rejections_list.count('\n- ') == 1 else rejections_list
             feedback(message=f"@{login}{rejections}{footer}")
-        if fw_rejections_list:
-            rejections = ' ' + fw_rejections_list.removeprefix("\n- ") if fw_rejections_list.count('\n- ') else fw_rejections_list
-            feedback(message=f"@{login}{rejections}{footer}", token='fp_github_token')
         return 'rejected'
 
+    def _approve(self, author, login):
+        oldstate = self.state
+        newstate = RPLUS.get(self.state)
+        msg = None
+        if not author.email:
+            msg = "I must know your email before you can review PRs. Please contact an administrator."
+        elif not newstate:
+            msg = "this PR is already reviewed, reviewing it again is useless."
+        else:
+            self.reviewed_by = author
+        _logger.debug(
+            "r+ on %s by %s (%s->%s) status=%s message? %s",
+            self.display_name, author.github_login,
+            oldstate, newstate or oldstate,
+            self.status, self.status == 'failure'
+        )
+        if self.status == 'failure':
+            # the normal infrastructure is for failure and
+            # prefixes messages with "I'm sorry"
+            self.env.ref("runbot_merge.command.approve.failure")._send(
+                repository=self.repository,
+                pull_request=self.number,
+                format_args={'user': login, 'pr': self},
+            )
+        return msg
+
+    def message_post(self, **kw):
+        if author := self.env.cr.precommit.data.get('change-author'):
+            kw['author_id'] = author
+        if message := self.env.cr.precommit.data.get('change-message'):
+            kw['body'] = html_escape(message)
+        return super().message_post(**kw)
+
+    def _message_log(self, **kw):
+        if author := self.env.cr.precommit.data.get('change-author'):
+            kw['author_id'] = author
+        if message := self.env.cr.precommit.data.get('change-message'):
+            kw['body'] = html_escape(message)
+        return super()._message_log(**kw)
+
     def _pr_acl(self, user):
         if not self:
             return ACL(False, False, False)
@@ -796,29 +900,67 @@ class PullRequests(models.Model):
         # could have two PRs (e.g. one open and one closed) at least
         # temporarily on the same head, or on the same head with different
         # targets
-        failed = self.browse(())
+        updateable = self.filtered(lambda p: p.state != 'merged')
+        updateable.statuses = statuses
+        for pr in updateable:
+            if pr.status == "failure":
+                statuses = json.loads(pr.statuses_full)
+                for ci in pr.repository.status_ids._for_pr(pr).mapped('context'):
+                    status = statuses.get(ci) or {'state': 'pending'}
+                    if status['state'] in ('error', 'failure'):
+                        pr._notify_ci_new_failure(ci, status)
+
+    def modified(self, fnames, create=False, before=False):
+        """ By default, Odoo can't express recursive *dependencies* which is
+        exactly what we need for statuses: they depend on the current PR's
+        overrides, and the parent's overrides, and *its* parent's overrides, ...
+
+        One option would be to create a stored computed field which accumulates
+        the overrides as *fields* can be recursive, but...
+        """
+        if 'overrides' in fnames:
+            descendants_or_self = self.concat(*self._iter_descendants())
+            self.env.add_to_compute(self._fields['status'], descendants_or_self)
+            self.env.add_to_compute(self._fields['statuses_full'], descendants_or_self)
+            self.env.add_to_compute(self._fields['state'], descendants_or_self)
+        super().modified(fnames, create, before)
+
+    @api.depends(
+        'statuses', 'overrides', 'target', 'parent_id',
+        'repository.status_ids.context',
+        'repository.status_ids.branch_filter',
+        'repository.status_ids.prs',
+    )
+    def _compute_statuses(self):
         for pr in self:
-            required = pr.repository.status_ids._for_pr(pr).mapped('context')
-            sts = {**statuses, **pr._get_overrides()}
+            statuses = {**json.loads(pr.statuses), **pr._get_overrides()}
 
-            success = True
-            for ci in required:
-                status = sts.get(ci) or {'state': 'pending'}
-                result = status['state']
-                if result == 'success':
-                    continue
+            pr.statuses_full = json.dumps(statuses, indent=4)
+
+            st = 'success'
+            for ci in pr.repository.status_ids._for_pr(pr):
+                v = (statuses.get(ci.context) or {'state': 'pending'})['state']
+                if v in ('error', 'failure'):
+                    st = 'failure'
+                    break
+                if v == 'pending':
+                    st = 'pending'
+            pr.status = st
+
+    # closed, merged, error should be exclusive, so this should probably be a selection
+    @api.depends("status", "reviewed_by", 'merge_date', "closed", "error")
+    def _compute_state(self):
+        for pr in self:
+            if pr.merge_date:
+                pr.state = 'merged'
+            elif pr.closed:
+                pr.state = "closed"
+            elif pr.error:
+                pr.state = "error"
+            else:
+                states = ("opened", "approved", "validated", "ready")
+                pr.state = states[bool(pr.reviewed_by) | ((pr.status == "success") << 1)]
 
-                success = False
-                if result in ('error', 'failure'):
-                    failed |= pr
-                    pr._notify_ci_new_failure(ci, status)
-            if success:
-                oldstate = pr.state
-                if oldstate == 'opened':
-                    pr.state = 'validated'
-                elif oldstate == 'approved':
-                    pr.state = 'ready'
-        return failed
 
     def _notify_ci_new_failure(self, ci, st):
         prev = json.loads(self.previous_failure)
@@ -865,6 +1007,18 @@ class PullRequests(models.Model):
             )
 
     def _auto_init(self):
+        for field in self._fields.values():
+            if not isinstance(field, fields.Selection) or field.column_type[0] == 'varchar':
+                continue
+
+            t = field.column_type[1]
+            self.env.cr.execute("SELECT 1 FROM pg_type WHERE typname = %s", [t])
+            if not self.env.cr.rowcount:
+                self.env.cr.execute(
+                    f"CREATE TYPE {t} AS ENUM %s",
+                    [tuple(s for s, _ in field.selection)]
+                )
+
         super(PullRequests, self)._auto_init()
         # incorrect index: unique(number, target, repository).
         tools.drop_index(self._cr, 'runbot_merge_unique_pr_per_target', self._table)
@@ -885,7 +1039,7 @@ class PullRequests(models.Model):
     def create(self, vals):
         pr = super().create(vals)
         c = self.env['runbot_merge.commit'].search([('sha', '=', pr.head)])
-        pr._validate(json.loads(c.statuses or '{}'))
+        pr._validate(c.statuses or '{}')
 
         if pr.state not in ('closed', 'merged'):
             self.env.ref('runbot_merge.pr.created')._send(
@@ -911,7 +1065,7 @@ class PullRequests(models.Model):
             ], limit=1)
 
         return self.env['runbot_merge.pull_requests'].create({
-            'state': 'opened' if description['state'] == 'open' else 'closed',
+            'closed': description['state'] != 'open',
             'number': description['number'],
             'label': repo._remap_label(description['head']['label']),
             'author': author.id,
@@ -926,30 +1080,65 @@ class PullRequests(models.Model):
     def write(self, vals):
         if vals.get('squash'):
             vals['merge_method'] = False
-        prev = None
-        if 'target' in vals or 'message' in vals:
-            prev = {
-                pr.id: {'target': pr.target, 'message': pr.message}
-                for pr in self
-            }
+        fields = []
+        canceler = vals.get('cancel_staging') or any(p.cancel_staging for p in self)
+        if canceler:
+            fields.append('state')
+            fields.append('skipchecks')
+        if 'target' in vals:
+            fields.append('target')
+        if 'message' in vals:
+            fields.append('message')
+        prev = {pr.id: {field: pr[field] for field in fields} for pr in self}
 
+        if vals.get('state') == 'ready':
+            # skip checks anyway
+            vals['skipchecks'] = True
+            # if the state is forced to ready, set current user as reviewer
+            # and override all statuses
+            vals.setdefault('reviewed_by', self.env.user.partner_id.id)
+            # override all known statuses just for safety
+            vals.setdefault('overrides', json.dumps({
+                st.context: {
+                    'state': 'success',
+                    'target_url': None,
+                    'description': f"Forced by @{self.env.user.partner_id.github_login}",
+                }
+                for st in self.env['runbot_merge.repository.status'].search([
+                    ('prs', '=', True),
+                ])
+            }))
+        if vals.get('closed'):
+            vals['reviewed_by'] = False
         w = super().write(vals)
 
         newhead = vals.get('head')
         if newhead:
             c = self.env['runbot_merge.commit'].search([('sha', '=', newhead)])
-            self._validate(json.loads(c.statuses or '{}'))
+            self._validate(c.statuses or '{}')
 
-        if prev:
-            for pr in self:
-                old_target = prev[pr.id]['target']
+        for pr in self:
+            old = prev[pr.id]
+            if canceler:
+                def ready(pr):
+                    return pr['state'] == 'ready'\
+                       or (pr['skipchecks'] and pr['state'] != 'error')
+                if pr.cancel_staging and not ready(old) and ready(pr):
+                    if old['state'] == 'error': # error -> ready gets a bespok message
+                        pr.target.active_staging_id.cancel(f"retrying {pr.display_name}")
+                    else:
+                        pr.target.active_staging_id.cancel(f"{pr.display_name} became ready")
+
+            if 'target' in vals:
+                old_target = old['target']
                 if pr.target != old_target:
                     pr.unstage(
                         "target (base) branch was changed from %r to %r",
                         old_target.display_name, pr.target.display_name,
                     )
-                old_message = prev[pr.id]['message']
-                if pr.merge_method not in (False, 'rebase-ff') and pr.message != old_message:
+
+            if 'message' in vals:
+                if pr.merge_method not in (False, 'rebase-ff') and pr.message != old['message']:
                     pr.unstage("merge message updated")
         return w
 
@@ -982,8 +1171,6 @@ class PullRequests(models.Model):
               bool_or(pr.state = 'ready' AND NOT pr.link_warned)
           -- one of the others should be unready
           AND bool_or(pr.state != 'ready')
-          -- but ignore batches with one of the prs at p0
-          AND bool_and(pr.priority != 0)
         """)
         for [ids] in self.env.cr.fetchall():
             prs = self.browse(ids)
@@ -1070,11 +1257,11 @@ class PullRequests(models.Model):
 
         self.env.cr.execute('''
         UPDATE runbot_merge_pull_requests
-        SET state = 'closed'
+        SET closed=True, state = 'closed', reviewed_by = null
         WHERE id = %s
         ''', [self.id])
         self.env.cr.commit()
-        self.modified(['state'])
+        self.modified(['closed', 'state', 'reviewed_by'])
         self.unstage("closed by %s", by)
         return True
 
@@ -1083,11 +1270,6 @@ RPLUS = {
     'opened': 'approved',
     'validated': 'ready',
 }
-RMINUS = {
-    'approved': 'opened',
-    'ready': 'validated',
-    'error': 'validated',
-}
 
 _TAGS = {
     False: set(),
@@ -1345,12 +1527,13 @@ class Commit(models.Model):
         for c in self.search([('to_check', '=', True)]):
             try:
                 c.to_check = False
-                st = json.loads(c.statuses)
                 pr = PRs.search([('head', '=', c.sha)])
                 if pr:
-                    pr._validate(st)
+                    self.env.cr.precommit.data['change-message'] =\
+                        f"statuses changed on {c.sha}"
+                    pr._validate(c.statuses)
 
-                stagings = Stagings.search([('head_ids.sha', '=', c.sha)])
+                stagings = Stagings.search([('head_ids.sha', '=', c.sha), ('state', '=', 'pending')])
                 if stagings:
                     stagings._validate()
             except Exception:
@@ -1545,8 +1728,10 @@ class Stagings(models.Model):
 
     def fail(self, message, prs=None):
         _logger.info("Staging %s failed: %s", self, message)
+        self.env.cr.precommit.data['change-message'] =\
+            f'staging {self.id} failed: {message}'
         prs = prs or self.batch_ids.prs
-        prs.write({'state': 'error'})
+        prs.error = True
         for pr in prs:
            self.env.ref('runbot_merge.pr.staging.fail')._send(
                repository=pr.repository,
@@ -1661,12 +1846,14 @@ class Stagings(models.Model):
                     'reason': str(e.__cause__ or e.__context__ or e)
                 })
             else:
+                self.env.cr.precommit.data['change-message'] =\
+                    f'staging {self.id} succeeded'
                 prs = self.mapped('batch_ids.prs')
                 logger.info(
                     "%s FF successful, marking %s as merged",
                     self, prs
                 )
-                prs.write({'state': 'merged'})
+                prs.merge_date = fields.Datetime.now()
 
                 pseudobranch = None
                 if self.target == project.branch_ids[:1]:
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index f426db7d..3329be23 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -65,19 +65,17 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
         return
 
     priority = rows[0][0]
-    if priority == 0 or priority == 1:
-        # p=0 take precedence over all else
-        # p=1 allows merging a fix inside / ahead of a split (e.g. branch
-        # is broken or widespread false positive) without having to cancel
-        # the existing staging
+    if priority == 'alone':
         batched_prs = [pr_ids for _, pr_ids in takewhile(lambda r: r[0] == priority, rows)]
     elif branch.split_ids:
         split_ids = branch.split_ids[0]
         _logger.info("Found split of PRs %s, re-staging", split_ids.mapped('batch_ids.prs'))
         batched_prs = [batch.prs for batch in split_ids.batch_ids]
         split_ids.unlink()
-    else: # p=2
-        batched_prs = [pr_ids for _, pr_ids in takewhile(lambda r: r[0] == priority, rows)]
+    else:
+        # priority, normal; priority = sorted ahead of normal, so always picked
+        # first as long as there's room
+        batched_prs = [pr_ids for _, pr_ids in rows]
 
     original_heads, staging_state = staging_setup(branch, batched_prs)
 
@@ -175,7 +173,7 @@ def ready_prs(for_branch: Branch) -> List[Tuple[int, PullRequests]]:
     env = for_branch.env
     env.cr.execute("""
     SELECT
-      min(pr.priority) as priority,
+      max(pr.priority) as priority,
       array_agg(pr.id) AS match
     FROM runbot_merge_pull_requests pr
     WHERE pr.target = any(%s)
@@ -191,8 +189,9 @@ def ready_prs(for_branch: Branch) -> List[Tuple[int, PullRequests]]:
             ELSE pr.label
         END
     HAVING
-        bool_or(pr.state = 'ready') or bool_or(pr.priority = 0)
-    ORDER BY min(pr.priority), min(pr.id)
+        bool_and(pr.state = 'ready')
+     OR (bool_or(pr.skipchecks) AND bool_and(pr.state != 'error'))
+    ORDER BY max(pr.priority) DESC, min(pr.id)
     """, [for_branch.ids])
     browse = env['runbot_merge.pull_requests'].browse
     return [(p, browse(ids)) for p, ids in env.cr.fetchall()]
@@ -408,7 +407,7 @@ def stage(pr: PullRequests, info: StagingSlice, related_prs: PullRequests) -> Tu
         diff.append(('Message', pr.message, msg))
 
     if invalid:
-        pr.write({**invalid, 'state': 'opened', 'head': pr_head})
+        pr.write({**invalid, 'reviewed_by': False, 'head': pr_head})
         raise exceptions.Mismatch(invalid, diff)
 
     if pr.reviewed_by and pr.reviewed_by.name == pr.reviewed_by.github_login:
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 5ddff161..cc4ff9f9 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -10,7 +10,7 @@ import requests
 from lxml import html
 
 import odoo
-from utils import _simple_init, seen, re_matches, get_partner, Commit, pr_page, to_pr, part_of
+from utils import _simple_init, seen, re_matches, get_partner, Commit, pr_page, to_pr, part_of, ensure_one
 
 
 @pytest.fixture
@@ -121,6 +121,35 @@ def test_trivial_flow(env, repo, page, users, config):
                              "\n\nSigned-off-by: {reviewer.formatted_email}"\
                              .format(repo=repo, reviewer=get_partner(env, users['reviewer']))
 
+    # reverse because the messages are in newest-to-oldest by default
+    # (as that's how you want to read them)
+    messages = reversed([
+        (m.author_id.display_name, m.body, list(zip(
+            m.tracking_value_ids.get_old_display_value(),
+            m.tracking_value_ids.get_new_display_value(),
+        )))
+        for m in pr_id.message_ids
+    ])
+
+    assert list(messages) == [
+        ('OdooBot', '<p>Pull Request created</p>', []),
+        ('OdooBot', f'<p>statuses changed on {c1}</p>', [('Opened', 'Validated')]),
+        # reviewer approved changing the state and setting reviewer as reviewer
+        # plus set merge method
+        ('Reviewer', '', [
+            ('Validated', 'Ready'),
+            ('', 'rebase and merge, using the PR as merge commit message'),
+            ('', 'Reviewer'),
+        ]),
+        # staging succeeded
+        (re_matches(r'.*'), f'<p>staging {st.id} succeeded</p>', [
+            # set merge date
+            (False, pr_id.merge_date + 'Z'),
+            # updated state
+            ('Ready', 'Merged'),
+        ]),
+    ]
+
 class TestCommitMessage:
     def test_commit_simple(self, env, repo, users, config):
         """ verify 'closes ...' is correctly added in the commit message
@@ -756,9 +785,17 @@ class TestPREdition:
             ('number', '=', prx.number)
         ]).target == branch_1
 
-    def test_retarget_update_commits(self, env, repo):
-        """ Retargeting a PR should update its commits count
+    def test_retarget_update_commits(self, env, project, repo):
+        """ Retargeting a PR should update its commits count, as well as follow
+        the new target's requirements
         """
+        project.repo_ids.write({
+            'status_ids': [
+                (5, 0, 0),
+                (0, 0, {'context': 'a', 'branch_filter': [('name', '=', 'master')]}),
+                (0, 0, {'context': 'b', 'branch_filter': [('name', '!=', 'master')]}),
+            ]
+        })
         branch_1 = env['runbot_merge.branch'].create({
             'name': '1.0',
             'project_id': env['runbot_merge.project'].search([]).id,
@@ -767,29 +804,35 @@ class TestPREdition:
 
         with repo:
             # master is 1 commit ahead of 1.0
-            m = repo.make_commit(None, 'initial', None, tree={'m': 'm'})
-            repo.make_ref('heads/1.0', m)
-            m2 = repo.make_commit(m, 'second', None, tree={'m': 'm2'})
-            repo.make_ref('heads/master', m2)
+            [m] = repo.make_commits(None, Commit('initial', tree={'m': 'm'}), ref='heads/1.0')
+            [m2] = repo.make_commits(m, Commit('second', tree={'m': 'm2'}), ref='heads/master')
 
             # the PR builds on master, but is errorneously targeted to 1.0
-            c = repo.make_commit(m2, 'first', None, tree={'m': 'm3'})
-            prx = repo.make_pr(title='title', body='body', target='1.0', head=c)
+            repo.make_commits(m2, Commit('first', tree={'m': 'm3'}), ref='heads/abranch')
+            prx = repo.make_pr(title='title', body='body', target='1.0', head='abranch')
+            repo.post_status('heads/abranch', 'success', 'a')
+        env.run_crons()
         pr = env['runbot_merge.pull_requests'].search([
             ('repository.name', '=', repo.name),
             ('number', '=', prx.number)
         ])
         assert not pr.squash
+        assert pr.status == 'pending'
+        assert pr.state == 'opened'
 
         with repo:
             prx.base = 'master'
         assert pr.target == master
         assert pr.squash
+        assert pr.status == 'success'
+        assert pr.state == 'validated'
 
         with repo:
             prx.base = '1.0'
         assert pr.target == branch_1
         assert not pr.squash
+        assert pr.status == 'pending'
+        assert pr.state == 'opened'
 
         # check if things also work right when modifying the PR then
         # retargeting (don't see why not but...)
@@ -845,6 +888,7 @@ def test_close_staged(env, repo, config, page):
         ('number', '=', prx.number),
     ])
     env.run_crons()
+    assert pr.reviewed_by
     assert pr.state == 'ready'
     assert pr.staging_id
 
@@ -856,6 +900,18 @@ def test_close_staged(env, repo, config, page):
     assert not env['runbot_merge.stagings'].search([])
     assert pr.state == 'closed'
     assert pr_page(page, prx).cssselect('.alert-light')
+    assert not pr.reviewed_by
+
+    with repo:
+        prx.open()
+    assert pr.state == 'validated'
+    assert not pr.reviewed_by
+
+    with repo:
+        prx.post_comment('hansen r+', config['role_reviewer']['token'])
+    assert pr.reviewed_by
+    pr.write({'closed': True})
+    assert not pr.reviewed_by
 
 def test_forward_port(env, repo, config):
     with repo:
@@ -2130,23 +2186,28 @@ class TestPRUpdate(object):
             repo.update_ref(prx.ref, c2, force=True)
         assert pr.head == c2
 
-    def test_reopen_update(self, env, repo):
+    def test_reopen_update(self, env, repo, config):
         with repo:
             m = repo.make_commit(None, 'initial', None, tree={'m': 'm'})
             repo.make_ref('heads/master', m)
 
             c = repo.make_commit(m, 'fist', None, tree={'m': 'c1'})
             prx = repo.make_pr(title='title', body='body', target='master', head=c)
+            prx.post_comment("hansen r+", config['role_reviewer']['token'])
 
         pr = to_pr(env, prx)
+        assert pr.state == 'approved'
+        assert pr.reviewed_by
         with repo:
             prx.close()
         assert pr.state == 'closed'
         assert pr.head == c
+        assert not pr.reviewed_by
 
         with repo:
             prx.open()
         assert pr.state == 'opened'
+        assert not pr.reviewed_by
 
         with repo:
             c2 = repo.make_commit(c, 'first', None, tree={'m': 'cc'})
@@ -2393,6 +2454,7 @@ class TestPRUpdate(object):
         env.run_crons('runbot_merge.process_updated_commits')
         assert pr_id.message == 'title\n\nbody'
         assert pr_id.state == 'ready'
+        old_reviewer = pr_id.reviewed_by
 
         # TODO: find way to somehow skip / ignore the update_ref?
         with repo:
@@ -2413,10 +2475,12 @@ class TestPRUpdate(object):
         # in a "ready" state
         pr_id.write({
             'head': c,
-            'state': 'ready',
+            'reviewed_by': old_reviewer.id,
             'message': "Something else",
             'target': other.id,
         })
+        assert pr_id.head == c
+        assert pr_id.state == "ready"
 
         env.run_crons()
 
@@ -2425,8 +2489,8 @@ class TestPRUpdate(object):
         assert pr_id.head == c2
         assert pr_id.message == 'title\n\nbody'
         assert pr_id.target.name == 'master'
-        assert pr.comments[-1]['body'] == """\
-@{} @{} we apparently missed updates to this PR and tried to stage it in a state \
+        assert pr.comments[-1]['body'] == f"""\
+@{users['user']} we apparently missed updates to this PR and tried to stage it in a state \
 which might not have been approved.
 
 The properties Head, Target, Message were not correctly synchronized and have been updated.
@@ -2435,8 +2499,8 @@ The properties Head, Target, Message were not correctly synchronized and have be
 
 ```diff
   Head:
-- {}
-+ {}
+- {c}
++ {c2}
   
   Target branch:
 - somethingelse
@@ -2454,7 +2518,7 @@ The properties Head, Target, Message were not correctly synchronized and have be
 Note that we are unable to check the properties Merge Method, Overrides, Draft.
 
 Please check and re-approve.
-""".format(users['user'], users['reviewer'], c, c2)
+"""
 
         # if the head commit doesn't change, that part should still be valid
         with repo:
@@ -2465,8 +2529,8 @@ Please check and re-approve.
 
         assert pr_id.message == 'title\n\nbody'
         assert pr_id.state == 'validated'
-        assert pr.comments[-1]['body'] == """\
-@{} @{} we apparently missed updates to this PR and tried to stage it in a state \
+        assert pr.comments[-1]['body'] == f"""\
+@{users['user']} we apparently missed updates to this PR and tried to stage it in a state \
 which might not have been approved.
 
 The properties Message were not correctly synchronized and have been updated.
@@ -2486,11 +2550,11 @@ The properties Message were not correctly synchronized and have been updated.
 Note that we are unable to check the properties Merge Method, Overrides, Draft.
 
 Please check and re-approve.
-""".format(users['user'], users['reviewer'])
+"""
 
         pr_id.write({
             'head': c,
-            'state': 'ready',
+            'reviewed_by': old_reviewer.id,
             'message': "Something else",
             'target': other.id,
             'draft': True,
@@ -2695,6 +2759,9 @@ class TestBatching(object):
     def test_batching_pressing(self, env, repo, config):
         """ "Pressing" PRs should be selected before normal & batched together
         """
+        # by limiting the batch size to 3 we allow both high-priority PRs, but 
+        # a single normal priority one
+        env['runbot_merge.project'].search([]).batch_limit = 3
         with repo:
             m = repo.make_commit(None, 'initial', None, tree={'a': 'some content'})
             repo.make_ref('heads/master', m)
@@ -2704,51 +2771,47 @@ class TestBatching(object):
 
             pr11 = self._pr(repo, 'Pressing1', [{'x': 'x'}, {'y': 'y'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'])
             pr12 = self._pr(repo, 'Pressing2', [{'z': 'z'}, {'zz': 'zz'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'])
-            pr11.post_comment('hansen priority=1', config['role_reviewer']['token'])
-            pr12.post_comment('hansen priority=1', config['role_reviewer']['token'])
-
-        pr21, pr22, pr11, pr12 = prs = [to_pr(env, pr) for pr in [pr21, pr22, pr11, pr12]]
-        assert pr21.priority == pr22.priority == 2
-        assert pr11.priority == pr12.priority == 1
-
+            pr11.post_comment('hansen priority', config['role_reviewer']['token'])
+            pr12.post_comment('hansen priority', config['role_reviewer']['token'])
+        # necessary to project commit statuses onto PRs
         env.run_crons()
 
+        pr21, pr22, pr11, pr12 = prs = [to_pr(env, pr) for pr in [pr21, pr22, pr11, pr12]]
+        assert pr11.priority == pr12.priority == 'priority'
+        assert pr21.priority == pr22.priority == 'default'
         assert all(pr.state == 'ready' for pr in prs)
-        assert not pr21.staging_id
+
+        staging = ensure_one(env['runbot_merge.stagings'].search([]))
+        assert staging.pr_ids == pr11 | pr12 | pr21
         assert not pr22.staging_id
-        assert pr11.staging_id
-        assert pr12.staging_id
-        assert pr11.staging_id == pr12.staging_id
 
     def test_batching_urgent(self, env, repo, config):
         with repo:
             m = repo.make_commit(None, 'initial', None, tree={'a': 'some content'})
             repo.make_ref('heads/master', m)
 
-            pr21 = self._pr(repo, 'PR1', [{'a': 'AAA'}, {'b': 'BBB'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'])
-            pr22 = self._pr(repo, 'PR2', [{'c': 'CCC'}, {'d': 'DDD'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'])
-
             pr11 = self._pr(repo, 'Pressing1', [{'x': 'x'}, {'y': 'y'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'])
             pr12 = self._pr(repo, 'Pressing2', [{'z': 'z'}, {'zz': 'zz'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'])
-            pr11.post_comment('hansen priority=1', config['role_reviewer']['token'])
-            pr12.post_comment('hansen priority=1', config['role_reviewer']['token'])
+            pr11.post_comment('hansen NOW', config['role_reviewer']['token'])
+            pr12.post_comment('hansen NOW', config['role_reviewer']['token'])
 
-        # stage PR1
+        # stage current PRs
         env.run_crons()
-        p_11, p_12, p_21, p_22 = \
-            [to_pr(env, pr) for pr in [pr11, pr12, pr21, pr22]]
-        assert not p_21.staging_id or p_22.staging_id
-        assert p_11.staging_id and p_12.staging_id
-        assert p_11.staging_id == p_12.staging_id
-        staging_1 = p_11.staging_id
+        p_11, p_12 = \
+            [to_pr(env, pr) for pr in [pr11, pr12]]
+        sm_all = p_11 | p_12
+        staging_1 = sm_all.staging_id
+        assert staging_1
+        assert len(staging_1) == 1
 
         # no statuses run on PR0s
         with repo:
             pr01 = self._pr(repo, 'Urgent1', [{'n': 'n'}, {'o': 'o'}], user=config['role_user']['token'], reviewer=None, statuses=[])
-            pr01.post_comment('hansen priority=0 rebase-merge', config['role_reviewer']['token'])
+            pr01.post_comment('hansen NOW! rebase-merge', config['role_reviewer']['token'])
         p_01 = to_pr(env, pr01)
-        assert p_01.state == 'opened'
-        assert p_01.priority == 0
+        assert p_01.state == 'approved'
+        assert p_01.priority == 'alone'
+        assert p_01.skipchecks == True
 
         env.run_crons()
         # first staging should be cancelled and PR0 should be staged
@@ -2757,8 +2820,89 @@ class TestBatching(object):
         assert not p_11.staging_id and not p_12.staging_id
         assert p_01.staging_id
 
+        # make the staging fail
+        with repo:
+            repo.post_status('staging.master', 'failure', 'ci/runbot')
+        env.run_crons()
+        assert p_01.state == 'error'
+        assert not p_01.staging_id.active
+        staging_2 = ensure_one(sm_all.staging_id)
+        assert staging_2 != staging_1
+
+        with repo:
+            pr01.post_comment('hansen retry', config['role_reviewer']['token'])
+        env.run_crons()
+        # retry should have re-triggered cancel-staging
+        assert not staging_2.active
+        assert p_01.staging_id.active
+
+        # make the staging fail again
+        with repo:
+            repo.post_status('staging.master', 'failure', 'ci/runbot')
+        env.run_crons()
+
+        assert not p_01.staging_id.active
+        assert p_01.state == 'error'
+        staging_3 = ensure_one(sm_all.staging_id)
+        assert staging_3 != staging_2
+
+        # check that updating the PR resets it to ~ready
+        with repo:
+            repo.make_commits(
+                'heads/master',
+                Commit("urgent+", tree={'y': 'es'}),
+                ref="heads/Urgent1",
+            )
+        env.run_crons()
+        assert not staging_3.active
+        assert p_01.state == 'opened'
+        assert p_01.priority == 'alone'
+        assert p_01.skipchecks == True
+        assert p_01.staging_id.active
+
+        # r- should unstage, re-enable the checks and switch off staging
+        # cancellation, but leave the priority
+        with repo:
+            pr01.post_comment("hansen r-", config['role_reviewer']['token'])
+        env.run_crons()
+
+        staging_4 = ensure_one(sm_all.staging_id)
+        assert staging_4 != staging_3
+
+        assert not p_01.staging_id.active
+        assert p_01.state == 'opened'
+        assert p_01.priority == 'alone'
+        assert p_01.skipchecks == False
+        assert p_01.cancel_staging == False
+
+        p_01.cancel_staging = True
+        # FIXME: cancel_staging should only cancel when the PR is or
+        #        transitions to ready
+        # assert staging_4.active
+        # re-staging, should not be necessary
+        env.run_crons()
+
+        staging_5 = ensure_one(sm_all.staging_id)
+        assert staging_5.active
+        # cause the PR to become ready the normal way
+        with repo:
+            pr01.post_comment("hansen r+", config['role_reviewer']['token'])
+            repo.post_status(p_01.head, 'success', 'legal/cla')
+            repo.post_status(p_01.head, 'success', 'ci/runbot')
+        env.run_crons()
+
+        # a cancel_staging pr becoming ready should have cancelled the staging,
+        # and because the PR is `alone` it should... have been restaged alone,
+        # without the ready non-alone PRs
+        assert not sm_all.staging_id.active
+        assert p_01.staging_id.active
+        assert p_01.state == 'ready'
+        assert p_01.priority == 'alone'
+        assert p_01.skipchecks == False
+        assert p_01.cancel_staging == True
+
     def test_batching_urgenter_than_split(self, env, repo, config):
-        """ p=0 PRs should take priority over split stagings (processing
+        """ p=alone PRs should take priority over split stagings (processing
         of a staging having CI-failed and being split into sub-stagings)
         """
         with repo:
@@ -2789,7 +2933,7 @@ class TestBatching(object):
         # during restaging of pr1, create urgent PR
         with repo:
             pr0 = self._pr(repo, 'urgent', [{'a': 'a', 'b': 'b'}], user=config['role_user']['token'], reviewer=None, statuses=[])
-            pr0.post_comment('hansen priority=0', config['role_reviewer']['token'])
+            pr0.post_comment('hansen NOW!', config['role_reviewer']['token'])
         env.run_crons()
 
         # TODO: maybe just deactivate stagings instead of deleting them when canceling?
@@ -2810,7 +2954,7 @@ class TestBatching(object):
         # no statuses run on PR0s
         with repo:
             pr01 = self._pr(repo, 'Urgent1', [{'n': 'n'}, {'o': 'o'}], user=config['role_user']['token'], reviewer=None, statuses=[])
-            pr01.post_comment('hansen priority=0', config['role_reviewer']['token'])
+            pr01.post_comment('hansen NOW!', config['role_reviewer']['token'])
         p_01 = to_pr(env, pr01)
         p_01.state = 'error'
 
@@ -2871,7 +3015,7 @@ class TestBatching(object):
         env.run_crons('runbot_merge.process_updated_commits', 'runbot_merge.merge_cron', 'runbot_merge.staging_cron')
         assert pr2.state == 'merged'
 
-class TestReviewing(object):
+class TestReviewing:
     def test_reviewer_rights(self, env, repo, users, config):
         """Only users with review rights will have their r+ (and other
         attributes) taken in account
@@ -3048,23 +3192,23 @@ class TestReviewing(object):
         ])
 
         with repo:
-            prx.post_review('COMMENT', "hansen priority=1", config['role_reviewer']['token'])
-        assert pr.priority == 1
+            prx.post_review('COMMENT', "hansen priority", config['role_reviewer']['token'])
+        assert pr.priority == 'priority'
         assert pr.state == 'opened'
 
         with repo:
-            prx.post_review('APPROVE', "hansen priority=2", config['role_reviewer']['token'])
-        assert pr.priority == 2
+            prx.post_review('APPROVE', "hansen default", config['role_reviewer']['token'])
+        assert pr.priority == 'default'
         assert pr.state == 'opened'
 
         with repo:
-            prx.post_review('REQUEST_CHANGES', 'hansen priority=1', config['role_reviewer']['token'])
-        assert pr.priority == 1
+            prx.post_review('REQUEST_CHANGES', 'hansen priority', config['role_reviewer']['token'])
+        assert pr.priority == 'priority'
         assert pr.state == 'opened'
 
         with repo:
             prx.post_review('COMMENT', 'hansen r+', config['role_reviewer']['token'])
-        assert pr.priority == 1
+        assert pr.priority == 'priority'
         assert pr.state == 'approved'
 
     def test_no_email(self, env, repo, users, config, partners):
@@ -3101,6 +3245,28 @@ class TestReviewing(object):
         env.run_crons()
         assert to_pr(env, pr).state == 'approved'
 
+    def test_skipchecks(self, env, repo, users, config):
+        """Skipcheck makes the PR immediately ready (if it's not in error or
+        something)
+        """
+        with repo:
+            [m, _] = repo.make_commits(
+                None,
+                Commit("initial", tree={'m': 'm'}),
+                Commit("second", tree={"m2": "m2"}),
+                ref="heads/master"
+            )
+
+            [c1] = repo.make_commits(m, Commit('first', tree={'m': 'c1'}))
+            pr = repo.make_pr(title='title', target='master', head=c1)
+            pr.post_comment('hansen skipchecks', config['role_reviewer']['token'])
+        env.run_crons()
+
+        pr_id = to_pr(env, pr)
+        # assert pr_id.state == 'ready'
+        assert not pr_id.blocked
+        # since the pr is not blocked it should have been staged by the relevant cron
+        assert pr_id.staging_id
 
 class TestUnknownPR:
     """ Sync PRs initially looked excellent but aside from the v4 API not
@@ -3157,7 +3323,7 @@ class TestUnknownPR:
             (users['reviewer'], 'hansen r+'),
             (users['reviewer'], 'hansen r+'),
             seen(env, prx, users),
-            (users['user'], f"@{users['user']} @{users['reviewer']} I didn't know about this PR and had to "
+            (users['user'], f"@{users['user']} I didn't know about this PR and had to "
                             "retrieve its information, you may have to "
                             "re-approve it as I didn't see previous commands."),
         ]
@@ -3213,7 +3379,7 @@ class TestUnknownPR:
             # reviewer is set because fetch replays all the comments (thus
             # setting r+ and reviewer) but then syncs the head commit thus
             # unsetting r+ but leaving the reviewer
-            (users['user'], f"@{users['user']} @{users['reviewer']} I didn't know about this PR and had to retrieve "
+            (users['user'], f"@{users['user']} I didn't know about this PR and had to retrieve "
                             "its information, you may have to re-approve it "
                             "as I didn't see previous commands."),
         ]
@@ -3576,41 +3742,6 @@ class TestRMinus:
         assert pr2.state == 'validated', "state should have been reset"
         assert not env['runbot_merge.split'].search([]), "there should be no split left"
 
-    def test_rminus_p0(self, env, repo, config, users):
-        """ In and of itself r- doesn't do anything on p=0 since they bypass
-        approval, so unstage and downgrade to p=1.
-        """
-
-        with repo:
-            m = repo.make_commit(None, 'initial', None, tree={'m': 'm'})
-            repo.make_ref('heads/master', m)
-
-            c = repo.make_commit(m, 'first', None, tree={'m': 'c'})
-            prx = repo.make_pr(title='title', body=None, target='master', head=c)
-            repo.post_status(prx.head, 'success', 'ci/runbot')
-            repo.post_status(prx.head, 'success', 'legal/cla')
-            prx.post_comment('hansen p=0', config['role_reviewer']['token'])
-        env.run_crons()
-
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number),
-        ])
-        assert pr.priority == 0
-        assert pr.staging_id
-
-        with repo:
-            prx.post_comment('hansen r-', config['role_reviewer']['token'])
-        env.run_crons()
-        assert not pr.staging_id, "pr should have been unstaged"
-        assert pr.priority == 1, "priority should have been downgraded"
-        assert prx.comments == [
-            (users['reviewer'], 'hansen p=0'),
-            seen(env, prx, users),
-            (users['reviewer'], 'hansen r-'),
-            (users['user'], "PR priority reset to 1, as pull requests with priority 0 ignore review state."),
-        ]
-
 class TestComments:
     def test_address_method(self, repo, env, config):
         with repo:
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index bee94ef8..04134966 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -5,6 +5,8 @@ source branches).
 When preparing a staging, we simply want to ensure branch-matched PRs
 are staged concurrently in all repos
 """
+import functools
+import operator
 import time
 import xmlrpc.client
 
@@ -775,8 +777,8 @@ class TestMultiBatches:
             prs[2][0] | prs[2][1] | prs[3][0] | prs[3][1] | prs[4][0]
 
 def test_urgent(env, repo_a, repo_b, config):
-    """ Either PR of a co-dependent pair being p=0 leads to the entire pair
-    being prioritized
+    """ Either PR of a co-dependent pair being prioritised leads to the entire
+    pair being prioritized
     """
     with repo_a, repo_b:
         make_branch(repo_a, 'master', 'initial', {'a0': 'a'})
@@ -784,19 +786,31 @@ def test_urgent(env, repo_a, repo_b, config):
 
         pr_a = make_pr(repo_a, 'batch', [{'a1': 'a'}, {'a2': 'a'}], user=config['role_user']['token'], reviewer=None, statuses=[])
         pr_b = make_pr(repo_b, 'batch', [{'b1': 'b'}, {'b2': 'b'}], user=config['role_user']['token'], reviewer=None, statuses=[])
-        pr_c = make_pr(repo_a, 'C', [{'c1': 'c', 'c2': 'c'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'],)
+        pr_c = make_pr(repo_a, 'C', [{'c1': 'c', 'c2': 'c'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'])
 
         pr_a.post_comment('hansen rebase-merge', config['role_reviewer']['token'])
-        pr_b.post_comment('hansen rebase-merge p=0', config['role_reviewer']['token'])
+        pr_b.post_comment('hansen rebase-merge alone skipchecks', config['role_reviewer']['token'])
     env.run_crons()
-    # should have batched pr_a and pr_b despite neither being reviewed or
-    # approved
-    p_a, p_b = to_pr(env, pr_a), to_pr(env, pr_b)
-    p_c = to_pr(env, pr_c)
+
+    p_a, p_b, p_c = to_pr(env, pr_a), to_pr(env, pr_b), to_pr(env, pr_c)
+    assert not p_a.blocked
+    assert not p_b.blocked
+
+    assert p_a.staging_id and p_b.staging_id and p_a.staging_id == p_b.staging_id,\
+        "a and b should be staged despite neither beinbg reviewed or approved"
     assert p_a.batch_id and p_b.batch_id and p_a.batch_id == p_b.batch_id,\
         "a and b should have been recognised as co-dependent"
     assert not p_c.staging_id
 
+    with repo_a:
+        pr_a.post_comment('hansen r-', config['role_reviewer']['token'])
+    env.run_crons()
+    assert not p_b.staging_id.active, "should be unstaged"
+    assert p_b.priority == 'alone', "priority should not be affected anymore"
+    assert not p_b.skipchecks, "r- of linked pr should have un-skipcheck-ed this one"
+    assert p_a.blocked
+    assert p_b.blocked
+
 class TestBlocked:
     def test_merge_method(self, env, repo_a, config):
         with repo_a:
@@ -854,8 +868,8 @@ class TestBlocked:
     def test_linked_unready(self, env, repo_a, repo_b, config):
         """ Create a PR A linked to a non-ready PR B,
         * A is blocked by default
-        * A is not blocked if A.p=0
-        * A is not blocked if B.p=0
+        * A is not blocked if A.skipci
+        * A is not blocked if B.skipci
         """
         with repo_a, repo_b:
             make_branch(repo_a, 'master', 'initial', {'a0': 'a'})
@@ -868,13 +882,11 @@ class TestBlocked:
         pr_a = to_pr(env, a)
         assert pr_a.blocked
 
-        with repo_a: a.post_comment('hansen p=0', config['role_reviewer']['token'])
+        with repo_a: a.post_comment('hansen skipchecks', config['role_reviewer']['token'])
         assert not pr_a.blocked
+        pr_a.skipchecks = False
 
-        with repo_a: a.post_comment('hansen p=2', config['role_reviewer']['token'])
-        assert pr_a.blocked
-
-        with repo_b: b.post_comment('hansen p=0', config['role_reviewer']['token'])
+        with repo_b: b.post_comment('hansen skipchecks', config['role_reviewer']['token'])
         assert not pr_a.blocked
 
 def test_different_branches(env, project, repo_a, repo_b, config):
@@ -1126,6 +1138,8 @@ def test_freeze_complete(env, project, repo_a, repo_b, repo_c, users, config):
     * check that freeze goes through
     * check that reminder is shown
     * check that new branches are created w/ correct parent & commit info
+    * check that a PRs (freeze and bump) are part of synthetic stagings so
+      they're correctly accounted for in the change history
     """
     project.freeze_reminder = "Don't forget to like and subscribe"
 
@@ -1215,22 +1229,25 @@ def test_freeze_complete(env, project, repo_a, repo_b, repo_c, users, config):
     assert r['res_model'] == 'runbot_merge.project'
     assert r['res_id'] == project.id
 
+    release_pr_ids = functools.reduce(operator.add, release_prs.values())
     # stuff that's done directly
-    for pr_id in release_prs.values():
-        assert pr_id.state == 'merged'
+    assert all(pr_id.state == 'merged' for pr_id in release_pr_ids)
     assert pr_bump_id.state == 'merged'
 
     # stuff that's behind a cron
     env.run_crons()
 
+    # check again to be sure
+    assert all(pr_id.state == 'merged' for pr_id in release_pr_ids)
+    assert pr_bump_id.state == 'merged'
+
     assert pr_rel_a.state == "closed"
     assert pr_rel_a.base['ref'] == '1.1'
     assert pr_rel_b.state == "closed"
     assert pr_rel_b.base['ref'] == '1.1'
     assert pr_rel_c.state == "closed"
     assert pr_rel_c.base['ref'] == '1.1'
-    for pr_id in release_prs.values():
-        assert pr_id.target.name == '1.1'
+    assert all(pr_id.target.name == '1.1' for pr_id in release_pr_ids)
 
     assert pr_bump_a.state == 'closed'
     assert pr_bump_a.base['ref'] == 'master'
diff --git a/runbot_merge/tests/test_oddities.py b/runbot_merge/tests/test_oddities.py
index 755a3312..43448656 100644
--- a/runbot_merge/tests/test_oddities.py
+++ b/runbot_merge/tests/test_oddities.py
@@ -252,3 +252,27 @@ def test_merge_emptying_commits(env, project, make_repo, setreviewers, users, co
     assert pr3.comments[3:] == [
         (users['user'], f"{ping} unable to stage: results in an empty tree when merged, might be the duplicate of a merged PR.")
     ]
+
+def test_force_ready(env, make_repo, project, setreviewers, config):
+    repo = make_repo('repo')
+    project.write({'repo_ids': [(0, 0, {
+        'name': repo.name,
+        'group_id': False,
+        'required_statuses': 'default',
+    })]})
+    setreviewers(*project.repo_ids)
+
+    with repo:
+        [m] = repo.make_commits(None, Commit('initial', tree={'m': 'm'}), ref="heads/master")
+
+        [c] = repo.make_commits(m, Commit('first', tree={'m': 'c1'}), ref="heads/other")
+        pr = repo.make_pr(title='title', body='body', target='master', head=c)
+    env.run_crons()
+
+    pr_id = to_pr(env, pr)
+    pr_id.state = 'ready'
+
+    assert pr_id.state == 'ready'
+    reviewer = env['res.users'].browse([env._uid]).partner_id
+    assert pr_id.reviewed_by == reviewer
+    assert pr_id.overrides
diff --git a/runbot_merge/tests/test_project_toggles.py b/runbot_merge/tests/test_project_toggles.py
new file mode 100644
index 00000000..e69de29b
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index 1650bfd4..dc219101 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -129,35 +129,77 @@
                         <h1>
                             <field name="repository"/>#<field name="number"/>
                         </h1>
+                        <h2>
+                          <field name="state"/>
+                          <span attrs="{'invisible': [('blocked', '=', False)]}">
+                            (<field name="blocked"/>)
+                          </span>
+                        </h2>
                     </div>
+                    <!-- massively impactful status items -->
                     <group>
-                        <group>
+                      <group>
+                            <field name="skipchecks" widget="boolean_toggle"/>
+                      </group>
+                      <group>
+                            <field name="cancel_staging" widget="boolean_toggle"/>
+                      </group>
+                    </group>
+                    <!-- main PR metadata -->
+                    <group>
+                        <group name="metadata">
                             <field name="target"/>
-                            <field name="state"/>
-                            <field name="author"/>
-                        </group>
-                        <group>
                             <field name="label"/>
+                            <field name="author"/>
+                            <field name="head"/>
+                        </group>
+                        <group name="merging">
                             <field name="priority"/>
+                            <field name="merge_method"/>
                             <field name="squash"/>
+                            <field name="draft"/>
+                            <field string="Up To" name="limit_id"/>
+                            <field string="Forward-Port Policy" name="fw_policy"/>
                         </group>
                     </group>
-                    <group>
-                        <group colspan="4">
-                            <field name="head"/>
-                            <field name="statuses"/>
+                    <!-- influencers -->
+                    <group string="State">
+                        <group>
+                            <field name="merge_date"/>
+                            <field name="reviewed_by"/>
+                            <field name="error"/>
+                            <field name="closed"/>
                         </group>
-                        <group colspan="4">
+                        <group name="status">
+                            <field name="status"/>
+                            <field name="statuses"/>
                             <field name="overrides"/>
                         </group>
                     </group>
-                    <group>
-                        <group colspan="4" string="Message">
+                    <group string="Forward Porting">
+                      <group>
+                          <field string="Original PR" name="source_id"/>
+                          <field name="parent_id"/>
+                          <field
+                              attrs="{'invisible': [('parent_id', '=', False), ('source_id', '!=', False)]}"
+                              string="Detached because" name="detach_reason" readonly="1"/>
+                      </group>
+                      <group>
+                        <field name="forwardport_ids" nolabel="1" readonly="True">
+                          <tree>
+                            <field name="target" string="Branch"/>
+                            <field name="number"/>
+                          </tree>
+                        </field>
+                      </group>
+                    </group>
+                    <group string="Message">
+                        <group colspan="4">
                             <field name="message" nolabel="1"/>
                         </group>
                     </group>
-                    <group>
-                        <group colspan="4" string="Delegates">
+                    <group string="Delegates">
+                        <group colspan="4">
                             <field name="delegates" nolabel="1">
                                 <tree>
                                     <button type="object" name="get_formview_action" icon="fa-external-link"/>
@@ -168,6 +210,10 @@
                         </group>
                     </group>
                 </sheet>
+                <div class="oe_chatter">
+                    <field name="message_follower_ids" widget="mail_followers"/>
+                    <field name="message_ids" widget="mail_thread"/>
+                </div>
             </form>
         </field>
     </record>

From 9f54e6f209f97bfe17f1f1a3aa79f5f7f23e9269 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 29 Nov 2023 13:58:40 +0100
Subject: [PATCH 080/192] [ADD] runbot_merge: option to disable staging without
 cron

Because the mergebot crons are on such a tight scheduling, and just
them finding out they have nothing to do can take a while, disabling
them can be a chore. Disabling staging via the project is much less
likely to cause issues as the projects don't normally (or ever?) get
exclusively locked, so they can generally be written to at any moment.

Furthermore, if we ever get in a situation where we have multiple
active projects (not really the case currently, we have multiple
projects but only one is really active) it's less disruptive to
disable stagings on a single specific project.

Fixes #860
---
 .../changelog/2023-12/staging-shutdown.md     | 14 +++++++
 runbot_merge/models/project.py                |  2 +
 runbot_merge/models/pull_requests.py          |  4 +-
 runbot_merge/tests/test_project_toggles.py    | 42 +++++++++++++++++++
 runbot_merge/tests/test_staging.py            |  6 +--
 runbot_merge/views/runbot_merge_project.xml   |  1 +
 6 files changed, 65 insertions(+), 4 deletions(-)
 create mode 100644 runbot_merge/changelog/2023-12/staging-shutdown.md

diff --git a/runbot_merge/changelog/2023-12/staging-shutdown.md b/runbot_merge/changelog/2023-12/staging-shutdown.md
new file mode 100644
index 00000000..1b22a65c
--- /dev/null
+++ b/runbot_merge/changelog/2023-12/staging-shutdown.md
@@ -0,0 +1,14 @@
+ADD: stagings can now be disabled on a per-project basis
+
+Currently stopping stagings requires stopping the staging cron(s), which causes
+several issues:
+
+- the staging cron runs very often, so it can be difficult to find a window to
+  deactivate it (as the cron runner acquires an exclusive lock on the cron)
+- the staging cron is global, so it does not disable staging only on the
+  problematic project (to say nothing of branch) but on all of them
+
+The latter is not currently a huge issue as only one of the mergebot-tracked
+projects is ultra active (spreadsheet activity is on the order of a few 
+single-PR stagings a day), but the former is really annoying when trying to
+stop runaway broken stagings.
diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index 6f88e117..2dcbd54d 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -24,6 +24,7 @@ class Project(models.Model):
         help="Branches of all project's repos which are managed by the merge bot. Also "\
         "target branches of PR this project handles."
     )
+    staging_enabled = fields.Boolean(default=True)
 
     ci_timeout = fields.Integer(
         default=60, required=True, group_operator=None,
@@ -141,6 +142,7 @@ class Project(models.Model):
             ('active_staging_id', '=', False),
             ('active', '=', True),
             ('staging_enabled', '=', True),
+            ('project_id.staging_enabled', '=', True),
         ]):
             try:
                 with self.env.cr.savepoint(), \
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 158cff5a..eba14234 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1716,7 +1716,7 @@ class Stagings(models.Model):
     def cancel(self, reason, *args):
         self = self.filtered('active')
         if not self:
-            return
+            return False
 
         _logger.info("Cancelling staging %s: " + reason, self, *args)
         self.mapped('batch_ids').write({'active': False})
@@ -1725,6 +1725,7 @@ class Stagings(models.Model):
             'state': 'cancelled',
             'reason': reason % args,
         })
+        return True
 
     def fail(self, message, prs=None):
         _logger.info("Staging %s failed: %s", self, message)
@@ -1745,6 +1746,7 @@ class Stagings(models.Model):
             'state': 'failure',
             'reason': message,
         })
+        return True
 
     def try_splitting(self):
         batches = len(self.batch_ids)
diff --git a/runbot_merge/tests/test_project_toggles.py b/runbot_merge/tests/test_project_toggles.py
index e69de29b..6f858b36 100644
--- a/runbot_merge/tests/test_project_toggles.py
+++ b/runbot_merge/tests/test_project_toggles.py
@@ -0,0 +1,42 @@
+import pytest
+
+from utils import Commit, to_pr
+
+@pytest.fixture
+def repo(env, project, make_repo, users, setreviewers):
+    r = make_repo('repo')
+    project.write({'repo_ids': [(0, 0, {
+        'name': r.name,
+        'group_id': False,
+        'required_statuses': 'default',
+    })]})
+    setreviewers(*project.repo_ids)
+    return r
+
+def test_disable_staging(env, project, repo, config):
+    """In order to avoid issues of cron locking, as well as not disable staging
+    for every project when trying to freeze just one of them (cough cough), a
+    toggle is available on the project to skip staging for it.
+    """
+    with repo:
+        [m] = repo.make_commits(None, Commit("m", tree={"a": "1"}), ref="heads/master")
+
+        [c] = repo.make_commits(m, Commit("c", tree={"a": "2"}), ref="heads/other")
+        pr = repo.make_pr(title="whatever", target="master", head="other")
+        pr.post_comment("hansen r+", config["role_reviewer"]['token'])
+        repo.post_status(c, "success")
+    env.run_crons()
+
+    pr_id = to_pr(env, pr)
+    staging_1 = pr_id.staging_id
+    assert staging_1.active
+
+    project.staging_enabled = False
+    staging_1.cancel("because")
+
+    env.run_crons()
+
+    assert staging_1.active is False
+    assert staging_1.state == "cancelled"
+    assert not pr_id.staging_id.active,\
+        "should not be re-staged, because staging has been disabled"
diff --git a/runbot_merge/tests/test_staging.py b/runbot_merge/tests/test_staging.py
index c594cc76..8903df62 100644
--- a/runbot_merge/tests/test_staging.py
+++ b/runbot_merge/tests/test_staging.py
@@ -9,7 +9,7 @@ def repo(env, project, make_repo, users, setreviewers):
     project.write({'repo_ids': [(0, 0, {
         'name': r.name,
         'group_id': False,
-        'required_statuses': 'ci'
+        'required_statuses': 'default'
     })]})
     setreviewers(*project.repo_ids)
     return r
@@ -26,13 +26,13 @@ def test_staging_disabled_branch(env, project, repo, config):
         [c1] = repo.make_commits(master_commit, Commit("thing", tree={'a': '2'}), ref='heads/master-thing')
         master_pr = repo.make_pr(title="whatever", target="master", head="master-thing")
         master_pr.post_comment("hansen r+", config['role_reviewer']['token'])
-        repo.post_status(c1, 'success', 'ci')
+        repo.post_status(c1, 'success')
 
         [other_commit] = repo.make_commits(None, Commit("other", tree={'b': '1'}), ref='heads/other')
         [c2] = repo.make_commits(other_commit, Commit("thing", tree={'b': '2'}), ref='heads/other-thing')
         other_pr = repo.make_pr(title="whatever", target="other", head="other-thing")
         other_pr.post_comment("hansen r+", config['role_reviewer']['token'])
-        repo.post_status(c2, 'success', 'ci')
+        repo.post_status(c2, 'success')
     env.run_crons()
 
     assert to_pr(env, master_pr).staging_id, \
diff --git a/runbot_merge/views/runbot_merge_project.xml b/runbot_merge/views/runbot_merge_project.xml
index 5aa49d47..59ed0829 100644
--- a/runbot_merge/views/runbot_merge_project.xml
+++ b/runbot_merge/views/runbot_merge_project.xml
@@ -32,6 +32,7 @@
                             <field name="secret"/>
                         </group>
                         <group>
+                            <field name="staging_enabled" widget="boolean_toggle"/>
                             <field name="uniquifier"/>
                             <field name="ci_timeout"/>
                             <field name="batch_limit"/>

From c140701975e7dd9acab228ac01544fc66110000b Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 30 Nov 2023 12:27:09 +0100
Subject: [PATCH 081/192] [ADD] runbot_merge: support staging ready PRs over
 splits

Not sure it's going to be useful but it's hard to know if we can't
test it. The intent is mostly the ability to prioritize throughput (or
attempt to) during high-load events, if we can favour staging N
new batches over a split's N/2 we might be able to merge more crap.

But maybe not, we'll see, either way now it's here and seems to more
or less work.

Fixes #798
---
 .../changelog/2023-12/staging-priority.md     |  4 +
 runbot_merge/models/project.py                |  5 ++
 runbot_merge/models/stagings_create.py        | 68 +++++++++++---
 runbot_merge/tests/test_project_toggles.py    | 89 ++++++++++++++++++-
 runbot_merge/views/runbot_merge_project.xml   |  1 +
 5 files changed, 155 insertions(+), 12 deletions(-)
 create mode 100644 runbot_merge/changelog/2023-12/staging-priority.md

diff --git a/runbot_merge/changelog/2023-12/staging-priority.md b/runbot_merge/changelog/2023-12/staging-priority.md
new file mode 100644
index 00000000..35bcfec8
--- /dev/null
+++ b/runbot_merge/changelog/2023-12/staging-priority.md
@@ -0,0 +1,4 @@
+ADD: projects now know how to prioritise new PRs over splits
+
+While this likely has relatively low utility, we'll look at how it performs
+during periods of high throughput.
diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index 2dcbd54d..c0420f9a 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -25,6 +25,11 @@ class Project(models.Model):
         "target branches of PR this project handles."
     )
     staging_enabled = fields.Boolean(default=True)
+    staging_priority = fields.Selection([
+        ('default', "Splits over ready PRs"),
+        ('largest', "Largest of split and ready PRs"),
+        ('ready', "Ready PRs over split"),
+    ], default="default", required=True)
 
     ci_timeout = fields.Integer(
         default=60, required=True, group_operator=None,
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index 3329be23..cb31425d 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -61,21 +61,67 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
         for p, prs in ready_prs(for_branch=branch)
         if not any(prs.mapped('blocked'))
     ]
-    if not rows:
-        return
 
-    priority = rows[0][0]
+    def log(label, batches):
+        _logger.info(label, ', '.join(
+            p.display_name
+            for batch in batches
+            for p in batch
+        ))
+
+    priority = rows[0][0] if rows else None
     if priority == 'alone':
         batched_prs = [pr_ids for _, pr_ids in takewhile(lambda r: r[0] == priority, rows)]
-    elif branch.split_ids:
-        split_ids = branch.split_ids[0]
-        _logger.info("Found split of PRs %s, re-staging", split_ids.mapped('batch_ids.prs'))
-        batched_prs = [batch.prs for batch in split_ids.batch_ids]
-        split_ids.unlink()
+        log("staging high-priority PRs %s", batched_prs)
+    elif branch.project_id.staging_priority == 'default':
+        if branch.split_ids:
+            split_ids = branch.split_ids[0]
+            batched_prs = [batch.prs for batch in split_ids.batch_ids]
+            split_ids.unlink()
+            log("staging split PRs %s (prioritising splits)", batched_prs)
+        else:
+            # priority, normal; priority = sorted ahead of normal, so always picked
+            # first as long as there's room
+            batched_prs = [pr_ids for _, pr_ids in rows]
+            log("staging ready PRs %s (prioritising splits)", batched_prs)
+    elif branch.project_id.staging_priority == 'ready':
+        # splits are ready by definition, we need to exclude them from the
+        # ready rows otherwise we'll re-stage the splits so if an error is legit
+        # we cycle forever
+        # FIXME: once the batches are less shit, store this durably on the
+        #        batches and filter out when fetching readies (?)
+        split_batches = {batch.prs for batch in branch.split_ids.batch_ids}
+        ready = [pr_ids for _, pr_ids in rows if pr_ids not in split_batches]
+
+        if ready:
+            batched_prs = ready
+            log("staging ready PRs %s (prioritising ready)", batched_prs)
+        else:
+            split_ids = branch.split_ids[:1]
+            batched_prs = [batch.prs for batch in split_ids.batch_ids]
+            split_ids.unlink()
+            log("staging split PRs %s (prioritising ready)", batched_prs)
     else:
-        # priority, normal; priority = sorted ahead of normal, so always picked
-        # first as long as there's room
-        batched_prs = [pr_ids for _, pr_ids in rows]
+        assert branch.project_id.staging_priority == 'largest'
+        # splits are ready by definition, we need to exclude them from the
+        # ready rows otherwise ready always wins but we re-stage the splits, so
+        # if an error is legit we'll cycle forever
+        split_batches = {batch.prs for batch in branch.split_ids.batch_ids}
+        ready = [pr_ids for _, pr_ids in rows if pr_ids not in split_batches]
+
+        maxsplit = max(branch.split_ids, key=lambda s: len(s.batch_ids), default=branch.env['runbot_merge.split'])
+        _logger.info("largest split = %d, ready = %d", len(maxsplit.batch_ids), len(ready))
+        # bias towards splits if len(ready) = len(batch_ids)
+        if len(maxsplit.batch_ids) >= len(ready):
+            batched_prs = [batch.prs for batch in maxsplit.batch_ids]
+            maxsplit.unlink()
+            log("staging split PRs %s (prioritising largest)", batched_prs)
+        else:
+            batched_prs = ready
+            log("staging ready PRs %s (prioritising largest)", batched_prs)
+
+    if not batched_prs:
+        return
 
     original_heads, staging_state = staging_setup(branch, batched_prs)
 
diff --git a/runbot_merge/tests/test_project_toggles.py b/runbot_merge/tests/test_project_toggles.py
index 6f858b36..876c241e 100644
--- a/runbot_merge/tests/test_project_toggles.py
+++ b/runbot_merge/tests/test_project_toggles.py
@@ -1,6 +1,10 @@
+import functools
+from itertools import repeat
+
 import pytest
 
-from utils import Commit, to_pr
+from utils import Commit, to_pr, ensure_one
+
 
 @pytest.fixture
 def repo(env, project, make_repo, users, setreviewers):
@@ -40,3 +44,86 @@ def test_disable_staging(env, project, repo, config):
     assert staging_1.state == "cancelled"
     assert not pr_id.staging_id.active,\
         "should not be re-staged, because staging has been disabled"
+
+@pytest.mark.parametrize('mode,cutoff,second', [
+    # default mode, the second staging is the first half of the first staging
+    ('default', 2, [0]),
+    # splits are right-biased (the midpoint is rounded down), so for odd
+    # staging sizes the first split is the smaller one
+    ('default', 3, [0]),
+    # if the split results in ((1, 2), 1), largest stages the second
+    ('largest', 3, [1, 2]),
+    # if the split results in ((1, 1), 2), largest stages the ready PRs
+    ('largest', 2, [2, 3]),
+    # even if it's a small minority, ready selects the ready PR(s)
+    ('ready', 3, [3]),
+    ('ready', 2, [2, 3]),
+])
+def test_staging_priority(env, project, repo, config, mode, cutoff, second):
+    """By default, unless a PR is prioritised as "alone" splits take priority
+    over new stagings.
+
+    *However* to try and maximise throughput in trying times, it's possible to
+    configure the project to prioritise either the largest staging (between spit
+    and ready batches), or to just prioritise new stagings.
+    """
+    def select(prs, indices):
+        zero = env['runbot_merge.pull_requests']
+        filtered = (p for i, p in enumerate(prs) if i in indices)
+        return functools.reduce(lambda a, b: a | b, filtered, zero)
+
+    project.staging_priority = mode
+    # we need at least 3 PRs, two that we can split out, and one leftover
+    with repo:
+        [m] = repo.make_commits(None, Commit("m", tree={"ble": "1"}), ref="heads/master")
+
+        [c] = repo.make_commits(m, Commit("c", tree={"1": "1"}), ref="heads/pr1")
+        pr1 = repo.make_pr(title="whatever", target="master", head="pr1")
+
+        [c] = repo.make_commits(m, Commit("c", tree={"2": "2"}), ref="heads/pr2")
+        pr2 = repo.make_pr(title="whatever", target="master", head="pr2")
+
+        [c] = repo.make_commits(m, Commit("c", tree={"3": "3"}), ref="heads/pr3")
+        pr3 = repo.make_pr(title="whatever", target="master", head="pr3")
+
+        [c] = repo.make_commits(m, Commit("c", tree={"4": "4"}), ref="heads/pr4")
+        pr4 = repo.make_pr(title="whatever", target="master", head="pr4")
+
+    prs = [pr1, pr2, pr3, pr4]
+    pr_ids = functools.reduce(
+        lambda a, b: a | b,
+        map(to_pr, repeat(env), prs)
+    )
+    # ready the PRs for the initial staging (to split)
+    pre_cutoff = pr_ids[:cutoff]
+    with repo:
+        for pr, pr_id in zip(prs[:cutoff], pre_cutoff):
+            pr.post_comment('hansen r+', config['role_reviewer']['token'])
+            repo.post_status(pr_id.head, 'success')
+    env.run_crons()
+    # check they staged as expected
+    assert all(p.staging_id for p in pre_cutoff)
+    staging = ensure_one(env['runbot_merge.stagings'].search([]))
+    ensure_one(pre_cutoff.staging_id)
+
+    # ready the rest
+    with repo:
+        for pr, pr_id in zip(prs[cutoff:], pr_ids[cutoff:]):
+            pr.post_comment('hansen r+', config['role_reviewer']['token'])
+            repo.post_status(pr_id.head, 'success')
+    env.run_crons('runbot_merge.process_updated_commits')
+    assert not pr_ids.filtered(lambda p: p.blocked)
+
+    # trigger a split
+    with repo:
+        repo.post_status('staging.master', 'failure')
+    env.run_crons('runbot_merge.process_updated_commits', 'runbot_merge.merge_cron')
+    assert not staging.active
+    assert not env['runbot_merge.stagings'].search([]).active
+    assert env['runbot_merge.split'].search_count([]) == 2
+
+    env.run_crons()
+
+    # check that st.pr_ids are the PRs we expect
+    st = env['runbot_merge.stagings'].search([])
+    assert st.pr_ids == select(pr_ids, second)
diff --git a/runbot_merge/views/runbot_merge_project.xml b/runbot_merge/views/runbot_merge_project.xml
index 59ed0829..d7b9324f 100644
--- a/runbot_merge/views/runbot_merge_project.xml
+++ b/runbot_merge/views/runbot_merge_project.xml
@@ -33,6 +33,7 @@
                         </group>
                         <group>
                             <field name="staging_enabled" widget="boolean_toggle"/>
+                            <field name="staging_priority"/>
                             <field name="uniquifier"/>
                             <field name="ci_timeout"/>
                             <field name="batch_limit"/>

From 473f89f87d6b43e847ffe8a59441feb50b7f5513 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 19 Dec 2023 11:10:11 +0100
Subject: [PATCH 082/192] [CHG] *: persistent batches

This probably has latent bugs, and is only the start of the road to v2
(#789): PR batches are now created up-front (alongside the PR), with
PRs attached and detached as needed, hopefully such that things are
not broken (tests pass but...), this required a fair number of
ajustments to code not taking batches into account, or creating
batches on the fly.

`PullRequests.blocked` has also been updated to rely on the batch to
get its batch-mates, such that it can now be a stored field with the
right dependencies.

The next step is to better leverage this change:

- move cross-PR state up to the batch (e.g. skipchecks, priority, ...)
- add fw info to the batch, perform forward-ports batchwise in order
  to avoid redundant batch-selection work, and allow altering batches
  during fw (e.g. adding or removing PRs)
- use batches to select stagings
- maybe expose staging history of a batch?
---
 forwardport/models/forwardport.py      |  42 ++++----
 forwardport/models/project.py          |  46 ++++-----
 forwardport/tests/test_limit.py        |   1 +
 forwardport/tests/test_weird.py        |  20 ++--
 runbot_merge/models/__init__.py        |   1 +
 runbot_merge/models/batch.py           |  21 ++++
 runbot_merge/models/pull_requests.py   | 135 ++++++++++++-------------
 runbot_merge/models/stagings_create.py |  20 ++--
 runbot_merge/tests/test_multirepo.py   |   2 -
 runbot_merge/views/templates.xml       |   2 +-
 10 files changed, 147 insertions(+), 143 deletions(-)
 create mode 100644 runbot_merge/models/batch.py

diff --git a/forwardport/models/forwardport.py b/forwardport/models/forwardport.py
index 7530d764..ee3d9b25 100644
--- a/forwardport/models/forwardport.py
+++ b/forwardport/models/forwardport.py
@@ -84,32 +84,32 @@ class ForwardPortTasks(models.Model, Queue):
         sentry_sdk.set_tag('forward-porting', batch.prs.mapped('display_name'))
         newbatch = batch.prs._port_forward()
 
-        if newbatch:
-            _logger.info(
-                "Processing %s (from %s): %s (%s) -> %s (%s)",
-                self.id, self.source,
-                batch, batch.prs,
-                newbatch, newbatch.prs,
-            )
-            # insert new batch in ancestry sequence unless conflict (= no parent)
-            if self.source == 'insert':
-                for pr in newbatch.prs:
-                    if not pr.parent_id:
-                        break
-                    newchild = pr.search([
-                        ('parent_id', '=', pr.parent_id.id),
-                        ('id', '!=', pr.id),
-                    ])
-                    if newchild:
-                        newchild.parent_id = pr.id
-        else: # reached end of seq (or batch is empty)
+        if not newbatch:  # reached end of seq (or batch is empty)
             # FIXME: or configuration is fucky so doesn't want to FP (maybe should error and retry?)
             _logger.info(
-                "Processing %s (from %s): %s (%s) -> end of the sequence",
+                "Processed %s (from %s): %s (%s) -> end of the sequence",
                 self.id, self.source,
                 batch, batch.prs
             )
-        batch.active = False
+            return
+
+        _logger.info(
+            "Processed %s (from %s): %s (%s) -> %s (%s)",
+            self.id, self.source,
+            batch, batch.prs,
+            newbatch, newbatch.prs,
+        )
+        # insert new batch in ancestry sequence unless conflict (= no parent)
+        if self.source == 'insert':
+            for pr in newbatch.prs:
+                if not pr.parent_id:
+                    break
+                newchild = pr.search([
+                    ('parent_id', '=', pr.parent_id.id),
+                    ('id', '!=', pr.id),
+                ])
+                if newchild:
+                    newchild.parent_id = pr.id
 
 
 class UpdateQueue(models.Model, Queue):
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 7b856893..e7c15df2 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -157,11 +157,7 @@ class Project(models.Model):
             # the parents linked list, so it has a special type
             for _, cs in groupby(candidates, key=lambda p: p.label):
                 self.env['forwardport.batches'].create({
-                    'batch_id': self.env['runbot_merge.batch'].create({
-                        'target': before[-1].id,
-                        'prs': [(4, c.id, 0) for c in cs],
-                        'active': False,
-                    }).id,
+                    'batch_id': cs[0].batch_id.id,
                     'source': 'insert',
                 })
 
@@ -321,7 +317,7 @@ class PullRequests(models.Model):
         addendum = ''
         # check if tip was queued for forward porting, try to cancel if we're
         # supposed to stop here
-        if real_limit == tip.target and (task := self.env['forwardport.batches'].search([('batch_id', 'in', tip.batch_ids.ids)])):
+        if real_limit == tip.target and (task := self.env['forwardport.batches'].search([('batch_id', '=', tip.batch_id.id)])):
             try:
                 with self.env.cr.savepoint():
                     self.env.cr.execute(
@@ -343,13 +339,13 @@ class PullRequests(models.Model):
             # resume
             if tip.state == 'merged':
                 self.env['forwardport.batches'].create({
-                    'batch_id': tip.batch_ids.sorted('id')[-1].id,
+                    'batch_id': tip.batch_id.id,
                     'source': 'fp' if tip.parent_id else 'merge',
                 })
                 resumed = tip
             else:
                 # reactivate batch
-                tip.batch_ids.sorted('id')[-1].active = True
+                tip.batch_id.active = True
                 resumed = tip._schedule_fp_followup()
             if resumed:
                 addendum += f', resuming forward-port stopped at {tip.display_name}'
@@ -411,18 +407,21 @@ class PullRequests(models.Model):
                 _logger.info('-> wrong state %s (%s)', pr.display_name, pr.state)
                 continue
 
-            # check if we've already forward-ported this branch:
-            # it has a batch without a staging
-            batch = self.env['runbot_merge.batch'].with_context(active_test=False).search([
-                ('staging_id', '=', False),
-                ('prs', 'in', pr.id),
-            ], limit=1)
-            # if the batch is inactive, the forward-port has been done *or*
-            # the PR's own forward port is in error, so bail
-            if not batch.active:
-                _logger.info('-> forward port done or in error (%s.active=%s)', batch, batch.active)
+            # check if we've already forward-ported this branch
+            source = pr.source_id or pr
+            next_target = source._find_next_target(pr)
+            if not next_target:
+                _logger.info("-> forward port done (no next target)")
                 continue
 
+            if n := self.search([
+                ('source_id', '=', source.id),
+                ('target', '=', next_target.id),
+            ], limit=1):
+                _logger.info('-> already forward-ported (%s)', n.display_name)
+                continue
+
+            batch = pr.batch_id
             # otherwise check if we already have a pending forward port
             _logger.info("%s %s %s", pr.display_name, batch, ', '.join(batch.mapped('prs.display_name')))
             if self.env['forwardport.batches'].search_count([('batch_id', '=', batch.id)]):
@@ -710,18 +709,9 @@ class PullRequests(models.Model):
                 'tags_add': labels,
             })
 
-        # batch the PRs so _validate can perform the followup FP properly
-        # (with the entire batch). If there are conflict then create a
-        # deactivated batch so the interface is coherent but we don't pickup
-        # an active batch we're never going to deactivate.
-        b = self.env['runbot_merge.batch'].create({
-            'target': target.id,
-            'prs': [(6, 0, new_batch.ids)],
-            'active': not has_conflicts,
-        })
         # try to schedule followup
         new_batch[0]._schedule_fp_followup()
-        return b
+        return new_batch.batch_id
 
     def _create_fp_branch(self, target_branch, fp_branch_name, cleanup):
         """ Creates a forward-port for the current PR to ``target_branch`` under
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index a75d586e..5d7d2313 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -367,6 +367,7 @@ def setci(*, source, repo, target, status='success'):
     in  ``repo``.
     """
     pr = source.search([('source_id', '=', source.id), ('target.name', '=', str(target))])
+    assert pr, f"could not find forward port of {source.display_name} to {target}"
     with repo:
         repo.post_status(pr.head, status)
 
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index a21f086f..8f7f8acd 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -191,19 +191,8 @@ def test_failed_staging(env, config, make_repo):
         prod.post_status('staging.c', 'failure', 'ci/runbot')
     env.run_crons()
 
-    pr3_head = env['runbot_merge.commit'].search([
-        ('sha', '=', pr3_id.head),
-    ])
-    assert len(pr3_head) == 1
-
-    assert not pr3_id.batch_id, "check that the PR indeed has no batch anymore"
-    assert not pr3_id.batch_ids.filtered(lambda b: b.active)
-
-    assert len(env['runbot_merge.batch'].search([
-        ('prs', 'in', pr3_id.id),
-        '|', ('active', '=', True),
-             ('active', '=', False),
-    ])) == 2, "check that there do exist batches"
+    pr3_head = env['runbot_merge.commit'].search([('sha', '=', pr3_id.head)])
+    assert pr3_head
 
     # send a new status to the PR, as if somebody had rebuilt it or something
     with prod:
@@ -213,6 +202,8 @@ def test_failed_staging(env, config, make_repo):
     assert pr3_head.to_check, "check that the commit was updated as to process"
     env.run_crons()
     assert not pr3_head.to_check, "check that the commit was processed"
+    assert pr3_id.state == 'ready'
+    assert pr3_id.staging_id
 
 class TestNotAllBranches:
     """ Check that forward-ports don't behave completely insanely when not all
@@ -788,6 +779,9 @@ def test_retarget_after_freeze(env, config, make_repo, users):
         prod.post_status('staging.bprime', 'success', 'legal/cla')
     env.run_crons()
 
+    # #2 batch 6 (???)
+    assert port_id.state == 'merged'
+
     new_pr_id = env['runbot_merge.pull_requests'].search([('state', 'not in', ('merged', 'closed'))])
     assert len(new_pr_id) == 1
     assert new_pr_id.parent_id == port_id
diff --git a/runbot_merge/models/__init__.py b/runbot_merge/models/__init__.py
index 6cbd92cf..18367537 100644
--- a/runbot_merge/models/__init__.py
+++ b/runbot_merge/models/__init__.py
@@ -2,6 +2,7 @@ from . import ir_actions
 from . import res_partner
 from . import project
 from . import pull_requests
+from . import batch
 from . import project_freeze
 from . import stagings_create
 from . import staging_cancel
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
new file mode 100644
index 00000000..aae71763
--- /dev/null
+++ b/runbot_merge/models/batch.py
@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from odoo import models, fields
+
+
+class Batch(models.Model):
+    """ A batch is a "horizontal" grouping of *codependent* PRs: PRs with
+    the same label & target but for different repositories. These are
+    assumed to be part of the same "change" smeared over multiple
+    repositories e.g. change an API in repo1, this breaks use of that API
+    in repo2 which now needs to be updated.
+    """
+    _name = _description = 'runbot_merge.batch'
+
+    target = fields.Many2one('runbot_merge.branch', required=True, index=True)
+    staging_ids = fields.Many2many('runbot_merge.stagings')
+    split_id = fields.Many2one('runbot_merge.split', index=True)
+
+    prs = fields.One2many('runbot_merge.pull_requests', 'batch_id')
+
+    active = fields.Boolean(default=True)
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index eba14234..ed61b6ef 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -400,9 +400,14 @@ class PullRequests(models.Model):
     ], compute='_compute_statuses', store=True, column_type=enum(_name, 'status'))
     previous_failure = fields.Char(default='{}')
 
-    batch_id = fields.Many2one('runbot_merge.batch', string="Active Batch", compute='_compute_active_batch', store=True)
-    batch_ids = fields.Many2many('runbot_merge.batch', string="Batches", context={'active_test': False})
-    staging_id = fields.Many2one(related='batch_id.staging_id', store=True)
+    batch_id = fields.Many2one('runbot_merge.batch', index=True)
+    staging_id = fields.Many2one('runbot_merge.stagings', compute='_compute_staging', store=True)
+
+    @api.depends('batch_id.staging_ids.active')
+    def _compute_staging(self):
+        for p in self:
+            p.staging_id = p.batch_id.staging_ids.filtered('active')
+
     commits_map = fields.Char(help="JSON-encoded mapping of PR commits to actually integrated commits. The integration head (either a merge commit or the PR's topmost) is mapped from the 'empty' pr commit (the key is an empty string, because you can't put a null key in json maps).", default='{}')
 
     link_warned = fields.Boolean(
@@ -411,7 +416,7 @@ class PullRequests(models.Model):
     )
 
     blocked = fields.Char(
-        compute='_compute_is_blocked',
+        compute='_compute_is_blocked', store=True,
         help="PR is not currently stageable for some reason (mostly an issue if status is ready)"
     )
 
@@ -520,28 +525,23 @@ class PullRequests(models.Model):
 
     @property
     def _linked_prs(self):
-        if re.search(r':patch-\d+', self.label):
-            return self.browse(())
-        if self.state == 'merged':
-            return self.with_context(active_test=False).batch_ids\
-                   .filtered(lambda b: b.staging_id.state == 'success')\
-                   .prs - self
-        return self.search([
-            ('target', '=', self.target.id),
-            ('label', '=', self.label),
-            ('state', 'not in', ('merged', 'closed')),
-        ]) - self
+        return self.batch_id.prs - self
 
-    # missing link to other PRs
-    @api.depends('state')
+    @api.depends(
+        'batch_id.prs.draft',
+        'batch_id.prs.squash',
+        'batch_id.prs.merge_method',
+        'batch_id.prs.state',
+        'batch_id.prs.skipchecks',
+    )
     def _compute_is_blocked(self):
         self.blocked = False
         requirements = (
             lambda p: not p.draft,
             lambda p: p.squash or p.merge_method,
             lambda p: p.state == 'ready' \
-                  or any(pr.skipchecks for pr in (p | p._linked_prs)) \
-                 and all(pr.state != 'error' for pr in (p | p._linked_prs))
+                  or any(p.batch_id.prs.mapped('skipchecks')) \
+                 and all(pr.state != 'error' for pr in p.batch_id.prs)
         )
         messages = ('is in draft', 'has no merge method', 'is not ready')
         for pr in self:
@@ -550,7 +550,7 @@ class PullRequests(models.Model):
 
             blocking, message = next((
                 (blocking, message)
-                for blocking in (pr | pr._linked_prs)
+                for blocking in pr.batch_id.prs
                 for requirement, message in zip(requirements, messages)
                 if not requirement(blocking)
             ), (None, None))
@@ -712,7 +712,7 @@ class PullRequests(models.Model):
                     else:
                         msg = self._approve(author, login)
                 case commands.Reject() if is_author:
-                    batch = self | self._linked_prs
+                    batch = self.batch_id.prs
                     if cancellers := batch.filtered('cancel_staging'):
                         cancellers.cancel_staging = False
                     if (skippers := batch.filtered('skipchecks')) or self.reviewed_by:
@@ -1035,8 +1035,22 @@ class PullRequests(models.Model):
             return 'staged'
         return self.state
 
+    def _get_batch(self, *, target, label):
+        Batches = self.env['runbot_merge.batch']
+        if re.search(r':patch-\d+$', label):
+            batch = Batches
+        else:
+            batch = Batches.search([
+                ('active', '=', True),
+                ('target', '=', target),
+                ('prs.label', '=', label),
+            ])
+        return batch or Batches.create({'target': target})
+
     @api.model
     def create(self, vals):
+        vals['batch_id'] = self._get_batch(target=vals['target'], label=vals['label']).id
+
         pr = super().create(vals)
         c = self.env['runbot_merge.commit'].search([('sha', '=', pr.head)])
         pr._validate(c.statuses or '{}')
@@ -1108,8 +1122,17 @@ class PullRequests(models.Model):
                     ('prs', '=', True),
                 ])
             }))
-        if vals.get('closed'):
-            vals['reviewed_by'] = False
+        match vals.get('closed'):
+            case True if not self.closed:
+                vals['reviewed_by'] = False
+                vals['batch_id'] = False
+                if len(self.batch_id.prs) == 1:
+                    self.env.cr.precommit.add(self.batch_id.unlink)
+            case False if self.closed:
+                vals['batch_id'] = self._get_batch(
+                    target=vals.get('target') or self.target.id,
+                    label=vals.get('label') or self.label,
+                )
         w = super().write(vals)
 
         newhead = vals.get('head')
@@ -1231,16 +1254,13 @@ class PullRequests(models.Model):
         """ If the PR is staged, cancel the staging. If the PR is split and
         waiting, remove it from the split (possibly delete the split entirely)
         """
-        split_batches = self.with_context(active_test=False).mapped('batch_ids').filtered('split_id')
-        if len(split_batches) > 1:
-            _logger.warning("Found a PR linked with more than one split batch: %s (%s)", self, split_batches)
-        for b in split_batches:
-            if len(b.split_id.batch_ids) == 1:
-                # only the batch of this PR -> delete split
-                b.split_id.unlink()
-            else:
-                # else remove this batch from the split
-                b.split_id = False
+        split = self.batch_id.split_id
+        if len(split.batch_ids) == 1:
+            # only the batch of this PR -> delete split
+            split.unlink()
+        else:
+            # else remove this batch from the split
+            self.batch_id.split_id = False
 
         self.staging_id.cancel('%s ' + reason, self.display_name, *args)
 
@@ -1248,21 +1268,29 @@ class PullRequests(models.Model):
         # ignore if the PR is already being updated in a separate transaction
         # (most likely being merged?)
         self.env.cr.execute('''
-        SELECT id, state FROM runbot_merge_pull_requests
+        SELECT id, state, batch_id FROM runbot_merge_pull_requests
         WHERE id = %s AND state != 'merged'
         FOR UPDATE SKIP LOCKED;
         ''', [self.id])
-        if not self.env.cr.fetchone():
+        r = self.env.cr.fetchone()
+        if not r:
             return False
 
         self.env.cr.execute('''
         UPDATE runbot_merge_pull_requests
-        SET closed=True, state = 'closed', reviewed_by = null
+        SET closed=True, reviewed_by = null
         WHERE id = %s
         ''', [self.id])
         self.env.cr.commit()
-        self.modified(['closed', 'state', 'reviewed_by'])
+        self.modified(['closed', 'reviewed_by', 'batch_id'])
         self.unstage("closed by %s", by)
+
+        # PRs should leave their batch when *closed*, and batches must die when
+        # no PR is linked
+        old_batch = self.batch_id
+        self.batch_id = False
+        if not old_batch.prs:
+            old_batch.unlink()
         return True
 
 # state changes on reviews
@@ -1571,10 +1599,7 @@ class Stagings(models.Model):
 
     target = fields.Many2one('runbot_merge.branch', required=True, index=True)
 
-    batch_ids = fields.One2many(
-        'runbot_merge.batch', 'staging_id',
-        context={'active_test': False},
-    )
+    batch_ids = fields.Many2many('runbot_merge.batch', context={'active_test': False})
     pr_ids = fields.One2many('runbot_merge.pull_requests', compute='_compute_prs')
     state = fields.Selection([
         ('success', 'Success'),
@@ -1972,34 +1997,6 @@ class Split(models.Model):
     target = fields.Many2one('runbot_merge.branch', required=True)
     batch_ids = fields.One2many('runbot_merge.batch', 'split_id', context={'active_test': False})
 
-class Batch(models.Model):
-    """ A batch is a "horizontal" grouping of *codependent* PRs: PRs with
-    the same label & target but for different repositories. These are
-    assumed to be part of the same "change" smeared over multiple
-    repositories e.g. change an API in repo1, this breaks use of that API
-    in repo2 which now needs to be updated.
-    """
-    _name = _description = 'runbot_merge.batch'
-
-    target = fields.Many2one('runbot_merge.branch', required=True, index=True)
-    staging_id = fields.Many2one('runbot_merge.stagings', index=True)
-    split_id = fields.Many2one('runbot_merge.split', index=True)
-
-    prs = fields.Many2many('runbot_merge.pull_requests')
-
-    active = fields.Boolean(default=True)
-
-    @api.constrains('target', 'prs')
-    def _check_prs(self):
-        for batch in self:
-            repos = self.env['runbot_merge.repository']
-            for pr in batch.prs:
-                if pr.target != batch.target:
-                    raise ValidationError("A batch and its PRs must have the same branch, got %s and %s" % (batch.target, pr.target))
-                if pr.repository in repos:
-                    raise ValidationError("All prs of a batch must have different target repositories, got a duplicate %s on %s" % (pr.repository, pr))
-                repos |= pr.repository
-
 
 class FetchJob(models.Model):
     _name = _description = 'runbot_merge.fetch_job'
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index cb31425d..f352176e 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -15,7 +15,8 @@ from werkzeug.datastructures import Headers
 
 from odoo import api, models, fields
 from odoo.tools import OrderedSet
-from .pull_requests import Branch, Stagings, PullRequests, Repository, Batch
+from .pull_requests import Branch, Stagings, PullRequests, Repository
+from .batch import Batch
 from .. import exceptions, utils, github, git
 
 WAIT_FOR_VISIBILITY = [10, 10, 10, 10]
@@ -285,8 +286,10 @@ def stage_batches(branch: Branch, batched_prs: List[PullRequests], staging_state
         if len(staged) >= batch_limit:
             break
 
+        assert len(bs := {p.batch_id for p in batch}) == 1,\
+            f"expected all PRs to have the same batch, got {bs}"
         try:
-            staged |= stage_batch(env, batch, staging_state)
+            staged |= stage_batch(env, batch[0].batch_id, staging_state)
         except exceptions.MergeError as e:
             pr = e.args[0]
             _logger.info("Failed to stage %s into %s", pr.display_name, branch.name, exc_info=True)
@@ -335,16 +338,18 @@ def parse_refs_smart(read: Callable[[int], bytes]) -> Iterator[Tuple[str, str]]:
 UNCHECKABLE = ['merge_method', 'overrides', 'draft']
 
 
-def stage_batch(env: api.Environment, prs: PullRequests, staging: StagingState) -> Batch:
+def stage_batch(env: api.Environment, batch: Batch, staging: StagingState):
     """Stages the batch represented by the ``prs`` recordset, onto the
     current corresponding staging heads.
 
     Alongside returning the newly created batch, updates ``staging[*].head``
     in-place on success. On failure, the heads should not be touched.
+
+    May return an empty recordset on some non-fatal failures.
     """
     new_heads: Dict[PullRequests, str] = {}
     pr_fields = env['runbot_merge.pull_requests']._fields
-    for pr in prs:
+    for pr in batch.prs:
         info = staging[pr.repository]
         _logger.info(
             "Staging pr %s for target %s; method=%s",
@@ -353,7 +358,7 @@ def stage_batch(env: api.Environment, prs: PullRequests, staging: StagingState)
         )
 
         try:
-            method, new_heads[pr] = stage(pr, info, related_prs=(prs - pr))
+            method, new_heads[pr] = stage(pr, info, related_prs=(batch.prs - pr))
             _logger.info(
                 "Staged pr %s to %s by %s: %s -> %s",
                 pr.display_name, pr.target.name, method,
@@ -382,10 +387,7 @@ def stage_batch(env: api.Environment, prs: PullRequests, staging: StagingState)
     # update meta to new heads
     for pr, head in new_heads.items():
         staging[pr.repository].head = head
-    return env['runbot_merge.batch'].create({
-        'target': prs[0].target.id,
-        'prs': [(4, pr.id, 0) for pr in prs],
-    })
+    return batch
 
 def format_for_difflib(items: Iterator[Tuple[str, object]]) -> Iterator[str]:
     """ Bit of a pain in the ass because difflib really wants
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 04134966..3c72b392 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -838,8 +838,6 @@ class TestBlocked:
         p = to_pr(env, pr)
         assert p.blocked
         with repo_b: b.close()
-        # FIXME: find a way for PR.blocked to depend on linked PR somehow so this isn't needed
-        p.invalidate_cache(['blocked'], [p.id])
         assert not p.blocked
 
     def test_linked_merged(self, env, repo_a, repo_b, config):
diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index 1a5fd4a1..7929208d 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -331,7 +331,7 @@
     <template id="view_pull_request_info_error">
         <div class="alert alert-danger">
             Error:
-            <span t-esc="pr.with_context(active_test=False).batch_ids[-1:].staging_id.reason">
+            <span t-esc="pr.with_context(active_test=False).batch_id.staging_ids[-1:].reason">
                 Unable to stage PR
             </span>
         </div>

From e910b8e85700ba740140dac2a48baf92f2272868 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 23 Jan 2024 15:57:41 +0100
Subject: [PATCH 083/192] [IMP] runbot_merge: move cross-pr properties to batch

---
 runbot_merge/models/batch.py           | 15 +++++++++++-
 runbot_merge/models/pull_requests.py   | 33 ++++++++++----------------
 runbot_merge/models/stagings_create.py | 11 +++------
 runbot_merge/tests/test_basic.py       |  2 +-
 4 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index aae71763..99977497 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -10,7 +10,9 @@ class Batch(models.Model):
     repositories e.g. change an API in repo1, this breaks use of that API
     in repo2 which now needs to be updated.
     """
-    _name = _description = 'runbot_merge.batch'
+    _name = 'runbot_merge.batch'
+    _description = "batch of pull request"
+    _inherit = ['mail.thread']
 
     target = fields.Many2one('runbot_merge.branch', required=True, index=True)
     staging_ids = fields.Many2many('runbot_merge.stagings')
@@ -19,3 +21,14 @@ class Batch(models.Model):
     prs = fields.One2many('runbot_merge.pull_requests', 'batch_id')
 
     active = fields.Boolean(default=True)
+
+    skipchecks = fields.Boolean(
+        string="Skips Checks",
+        default=False, tracking=True,
+        help="Forces entire batch to be ready, skips validation and approval",
+    )
+    cancel_staging = fields.Boolean(
+        string="Cancels Stagings",
+        default=False, tracking=True,
+        help="Cancels current staging on target branch when becoming ready"
+    )
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index ed61b6ef..b56835d6 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -327,16 +327,8 @@ class PullRequests(models.Model):
 
     closed = fields.Boolean(default=False, tracking=True)
     error = fields.Boolean(string="in error", default=False, tracking=True)
-    skipchecks = fields.Boolean(
-        string="Skips Checks",
-        default=False, tracking=True,
-        help="Forces entire batch to be ready, skips validation and approval",
-    )
-    cancel_staging = fields.Boolean(
-        string="Cancels Stagings",
-        default=False, tracking=True,
-        help="Cancels current staging on target branch when becoming ready"
-    )
+    skipchecks = fields.Boolean(related='batch_id.skipchecks')
+    cancel_staging = fields.Boolean(related='batch_id.cancel_staging')
     merge_date = fields.Datetime(tracking=True)
 
     state = fields.Selection([
@@ -532,7 +524,7 @@ class PullRequests(models.Model):
         'batch_id.prs.squash',
         'batch_id.prs.merge_method',
         'batch_id.prs.state',
-        'batch_id.prs.skipchecks',
+        'batch_id.skipchecks',
     )
     def _compute_is_blocked(self):
         self.blocked = False
@@ -540,7 +532,7 @@ class PullRequests(models.Model):
             lambda p: not p.draft,
             lambda p: p.squash or p.merge_method,
             lambda p: p.state == 'ready' \
-                  or any(p.batch_id.prs.mapped('skipchecks')) \
+                  or p.batch_id.skipchecks \
                  and all(pr.state != 'error' for pr in p.batch_id.prs)
         )
         messages = ('is in draft', 'has no merge method', 'is not ready')
@@ -712,20 +704,19 @@ class PullRequests(models.Model):
                     else:
                         msg = self._approve(author, login)
                 case commands.Reject() if is_author:
-                    batch = self.batch_id.prs
-                    if cancellers := batch.filtered('cancel_staging'):
-                        cancellers.cancel_staging = False
-                    if (skippers := batch.filtered('skipchecks')) or self.reviewed_by:
+                    if cancellers := self.batch_id.cancel_staging:
+                        self.batch_id.cancel_staging = False
+                    if self.batch_id.skipchecks or self.reviewed_by:
                         if self.error:
                             self.error = False
                         if self.reviewed_by:
                             self.reviewed_by = False
-                        if skippers:
-                            skippers.skipchecks = False
+                        if self.batch_id.skipchecks:
+                            self.batch_id.skipchecks = False
                             self.env.ref("runbot_merge.command.unapprove.p0")._send(
                                 repository=self.repository,
                                 pull_request=self.number,
-                                format_args={'user': login, 'pr': skippers[:1]},
+                                format_args={'user': login, 'pr': self.batch_id.prs[:1]},
                             )
                         self.unstage("unreviewed (r-) by %s", login)
                     else:
@@ -762,13 +753,13 @@ class PullRequests(models.Model):
                 case commands.Priority() if is_admin:
                     self.priority = str(command)
                 case commands.SkipChecks() if is_admin:
-                    self.skipchecks = True
+                    self.batch_id.skipchecks = True
                     self.reviewed_by = author
                     for p in self.batch_id.prs - self:
                         if not p.reviewed_by:
                             p.reviewed_by = author
                 case commands.CancelStaging() if is_admin:
-                    self.cancel_staging = True
+                    self.batch_id.cancel_staging = True
                     # FIXME: remove this when skipchecks properly affects state,
                     #        maybe: staging cancellation should then only occur
                     #        when a cancel_staging PR transitions to ready, or
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index f352176e..f7f6f971 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -223,21 +223,16 @@ def ready_prs(for_branch: Branch) -> List[Tuple[int, PullRequests]]:
       max(pr.priority) as priority,
       array_agg(pr.id) AS match
     FROM runbot_merge_pull_requests pr
+    JOIN runbot_merge_batch b ON (b.id = pr.batch_id)
     WHERE pr.target = any(%s)
       -- exclude terminal states (so there's no issue when
       -- deleting branches & reusing labels)
       AND pr.state != 'merged'
       AND pr.state != 'closed'
-    GROUP BY
-        pr.target,
-        CASE
-            WHEN pr.label SIMILAR TO '%%:patch-[[:digit:]]+'
-                THEN pr.id::text
-            ELSE pr.label
-        END
+    GROUP BY b.id
     HAVING
         bool_and(pr.state = 'ready')
-     OR (bool_or(pr.skipchecks) AND bool_and(pr.state != 'error'))
+     OR (bool_or(b.skipchecks) AND bool_and(pr.state != 'error'))
     ORDER BY max(pr.priority) DESC, min(pr.id)
     """, [for_branch.ids])
     browse = env['runbot_merge.pull_requests'].browse
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index cc4ff9f9..2d0bc440 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -2875,7 +2875,7 @@ class TestBatching(object):
         assert p_01.skipchecks == False
         assert p_01.cancel_staging == False
 
-        p_01.cancel_staging = True
+        p_01.batch_id.cancel_staging = True
         # FIXME: cancel_staging should only cancel when the PR is or
         #        transitions to ready
         # assert staging_4.active

From 21b5dd439b45c24c00563d86ace77fa727c097d9 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 29 Jan 2024 14:09:22 +0100
Subject: [PATCH 084/192] [CHG] runbot_merge: move merge_date to batch, remove
 active

- `merge_date` should be common to an entire batch, so move it there
- remove `Batch.active` which should probably have been removed when
  batches were made persistent (can eventually re-add as a proxy for
  `merge_date` being set maybe, but for now removing it seems a better
  way to catch mistakes)
- update various sites to use `Batch.merge_date` instead of
  `Batch.active`
---
 forwardport/models/project.py                 | 22 +++++++++++-----
 runbot_merge/models/batch.py                  |  3 +--
 .../models/project_freeze/__init__.py         |  6 ++---
 runbot_merge/models/pull_requests.py          | 26 +++++++++----------
 4 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index e7c15df2..a5c60d8f 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -258,11 +258,6 @@ class PullRequests(models.Model):
                         token_field='fp_github_token',
                         format_args={'pr': parent, 'child': p},
                     )
-        if vals.get('merge_date'):
-            self.env['forwardport.branch_remover'].create([
-                {'pr_id': p.id}
-                for p in self
-            ])
         # if we change the policy to skip CI, schedule followups on existing FPs
         if vals.get('fw_policy') == 'skipci' and self.state == 'merged':
             self.env['runbot_merge.pull_requests'].search([
@@ -344,8 +339,6 @@ class PullRequests(models.Model):
                 })
                 resumed = tip
             else:
-                # reactivate batch
-                tip.batch_id.active = True
                 resumed = tip._schedule_fp_followup()
             if resumed:
                 addendum += f', resuming forward-port stopped at {tip.display_name}'
@@ -982,6 +975,21 @@ stderr:
                 }
             )
 
+class Batch(models.Model):
+    _inherit = 'runbot_merge.batch'
+
+    def write(self, vals):
+        if vals.get('merge_date'):
+            self.env['forwardport.branch_remover'].create([
+                {'pr_id': p.id}
+                for b in self
+                if not b.merge_date
+                for p in b.prs
+            ])
+
+        super().write(vals)
+
+        return True
 
 # ordering is a bit unintuitive because the lowest sequence (and name)
 # is the last link of the fp chain, reasoning is a bit more natural the
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index 99977497..a9f6a3d1 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -20,8 +20,7 @@ class Batch(models.Model):
 
     prs = fields.One2many('runbot_merge.pull_requests', 'batch_id')
 
-    active = fields.Boolean(default=True)
-
+    merge_date = fields.Datetime(tracking=True)
     skipchecks = fields.Boolean(
         string="Skips Checks",
         default=False, tracking=True,
diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py
index 34758b7f..7215d762 100644
--- a/runbot_merge/models/project_freeze/__init__.py
+++ b/runbot_merge/models/project_freeze/__init__.py
@@ -339,10 +339,8 @@ class FreezeWizard(models.Model):
             )
 
         all_prs = self.release_pr_ids.pr_id | self.bump_pr_ids.pr_id
-        all_prs.write({
-            'merge_date': fields.Datetime.now(),
-            'reviewed_by': self.env.user.partner_id.id,
-        })
+        all_prs.batch_id.merge_date = fields.Datetime.now()
+        all_prs.reviewed_by = self.env.user.partner_id.id
         self.env['runbot_merge.pull_requests.feedback'].create([{
             'repository': pr.repository.id,
             'pull_request': pr.number,
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index b56835d6..eb28fe20 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -15,8 +15,7 @@ from typing import Optional, Union, List, Iterator, Tuple
 import sentry_sdk
 import werkzeug
 
-from odoo import api, fields, models, tools
-from odoo.exceptions import ValidationError
+from odoo import api, fields, models, tools, Command
 from odoo.osv import expression
 from odoo.tools import html_escape
 from . import commands
@@ -329,7 +328,12 @@ class PullRequests(models.Model):
     error = fields.Boolean(string="in error", default=False, tracking=True)
     skipchecks = fields.Boolean(related='batch_id.skipchecks')
     cancel_staging = fields.Boolean(related='batch_id.cancel_staging')
-    merge_date = fields.Datetime(tracking=True)
+    merge_date = fields.Datetime(
+        related='batch_id.merge_date',
+        inverse=lambda _: 1/0,
+        tracking=True,
+        store=True,
+    )
 
     state = fields.Selection([
         ('opened', 'Opened'),
@@ -939,10 +943,10 @@ class PullRequests(models.Model):
             pr.status = st
 
     # closed, merged, error should be exclusive, so this should probably be a selection
-    @api.depends("status", "reviewed_by", 'merge_date', "closed", "error")
+    @api.depends("status", "reviewed_by", 'batch_id.merge_date', "closed", "error")
     def _compute_state(self):
         for pr in self:
-            if pr.merge_date:
+            if pr.batch_id.merge_date:
                 pr.state = 'merged'
             elif pr.closed:
                 pr.state = "closed"
@@ -1032,7 +1036,7 @@ class PullRequests(models.Model):
             batch = Batches
         else:
             batch = Batches.search([
-                ('active', '=', True),
+                ('merge_date', '=', False),
                 ('target', '=', target),
                 ('prs.label', '=', label),
             ])
@@ -1735,7 +1739,6 @@ class Stagings(models.Model):
             return False
 
         _logger.info("Cancelling staging %s: " + reason, self, *args)
-        self.mapped('batch_ids').write({'active': False})
         self.write({
             'active': False,
             'state': 'cancelled',
@@ -1756,7 +1759,6 @@ class Stagings(models.Model):
                format_args={'pr': pr, 'message': message},
            )
 
-        self.batch_ids.write({'active': False})
         self.write({
             'active': False,
             'state': 'failure',
@@ -1772,15 +1774,14 @@ class Stagings(models.Model):
             # NB: batches remain attached to their original staging
             sh = self.env['runbot_merge.split'].create({
                 'target': self.target.id,
-                'batch_ids': [(4, batch.id, 0) for batch in h],
+                'batch_ids': [Command.link(batch.id) for batch in h],
             })
             st = self.env['runbot_merge.split'].create({
                 'target': self.target.id,
-                'batch_ids': [(4, batch.id, 0) for batch in t],
+                'batch_ids': [Command.link(batch.id) for batch in t],
             })
             _logger.info("Split %s to %s (%s) and %s (%s)",
                          self, h, sh, t, st)
-            self.batch_ids.write({'active': False})
             self.write({
                 'active': False,
                 'state': 'failure',
@@ -1871,7 +1872,7 @@ class Stagings(models.Model):
                     "%s FF successful, marking %s as merged",
                     self, prs
                 )
-                prs.merge_date = fields.Datetime.now()
+                self.batch_ids.merge_date = fields.Datetime.now()
 
                 pseudobranch = None
                 if self.target == project.branch_ids[:1]:
@@ -1893,7 +1894,6 @@ class Stagings(models.Model):
                             'tags_add': json.dumps([pseudobranch]),
                         })
             finally:
-                self.batch_ids.write({'active': False})
                 self.write({'active': False})
         elif self.state == 'failure' or self.is_timed_out():
             self.try_splitting()

From 9ddf01776810991e8fcaf0b68f0c9571cffc1603 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 21 May 2024 15:44:47 +0200
Subject: [PATCH 085/192] [CHG] *: move fw_policy from PR to batch

---
 forwardport/models/project.py        | 24 ++++++++++++++----------
 forwardport/tests/test_weird.py      |  4 ++--
 runbot_merge/models/batch.py         |  5 +++++
 runbot_merge/models/pull_requests.py |  2 +-
 runbot_merge/views/mergebot.xml      |  2 +-
 5 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index a5c60d8f..d4a30f21 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -38,8 +38,8 @@ from odoo.tools.misc import topological_sort, groupby, Reverse
 from odoo.tools.sql import reverse_order
 from odoo.tools.appdirs import user_cache_dir
 from odoo.addons.base.models.res_partner import Partner
-from odoo.addons.runbot_merge import git, utils
-from odoo.addons.runbot_merge.models.pull_requests import RPLUS, Branch
+from odoo.addons.runbot_merge import git
+from odoo.addons.runbot_merge.models.pull_requests import Branch
 from odoo.addons.runbot_merge.models.stagings_create import Message
 
 
@@ -258,12 +258,6 @@ class PullRequests(models.Model):
                         token_field='fp_github_token',
                         format_args={'pr': parent, 'child': p},
                     )
-        # if we change the policy to skip CI, schedule followups on existing FPs
-        if vals.get('fw_policy') == 'skipci' and self.state == 'merged':
-            self.env['runbot_merge.pull_requests'].search([
-                ('source_id', '=', self.id),
-                ('state', 'not in', ('closed', 'merged')),
-            ])._schedule_fp_followup()
         return r
 
     def _try_closing(self, by):
@@ -396,7 +390,7 @@ class PullRequests(models.Model):
             if not pr.parent_id:
                 _logger.info('-> no parent %s (%s)', pr.display_name, pr.parent_id)
                 continue
-            if not self.env.context.get('force_fw') and self.source_id.fw_policy != 'skipci' and pr.state not in ['validated', 'ready']:
+            if not self.env.context.get('force_fw') and self.source_id.batch_id.fw_policy != 'skipci' and pr.state not in ['validated', 'ready']:
                 _logger.info('-> wrong state %s (%s)', pr.display_name, pr.state)
                 continue
 
@@ -424,7 +418,7 @@ class PullRequests(models.Model):
             # check if batch-mate are all valid
             mates = batch.prs
             # wait until all of them are validated or ready
-            if not self.env.context.get('force_fw') and any(pr.source_id.fw_policy != 'skipci' and pr.state not in ('validated', 'ready') for pr in mates):
+            if not self.env.context.get('force_fw') and any(pr.source_id.batch_id.fw_policy != 'skipci' and pr.state not in ('validated', 'ready') for pr in mates):
                 _logger.info("-> not ready (%s)", [(pr.display_name, pr.state) for pr in mates])
                 continue
 
@@ -989,6 +983,16 @@ class Batch(models.Model):
 
         super().write(vals)
 
+        # if we change the policy to skip CI, schedule followups on existing FPs
+        # FIXME: although this should be managed by the command handler already,
+        #        this should probably allow setting the fw policy on any batch
+        #        of the sequence
+        if vals.get('fw_policy') == 'skipci' and self.merge_date:
+            self.env['runbot_merge.pull_requests'].search([
+                ('source_id', '=', self.prs[:1].id),
+                ('state', 'not in', ('closed', 'merged')),
+            ])._schedule_fp_followup()
+
         return True
 
 # ordering is a bit unintuitive because the lowest sequence (and name)
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index 8f7f8acd..0c2170b4 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -655,7 +655,7 @@ def test_skip_ci_all(env, config, make_repo):
     assert env['runbot_merge.pull_requests'].search([
         ('repository.name', '=', prod.name),
         ('number', '=', pr.number)
-    ]).fw_policy == 'skipci'
+    ]).batch_id.fw_policy == 'skipci'
 
     with prod:
         prod.post_status('staging.a', 'success', 'legal/cla')
@@ -695,7 +695,7 @@ def test_skip_ci_next(env, config, make_repo):
             'hansen fw=skipci',
             config['role_reviewer']['token']
         )
-    assert pr0_id.fw_policy == 'skipci'
+    assert pr0_id.batch_id.fw_policy == 'skipci'
     env.run_crons()
 
     _, _, pr2_id = env['runbot_merge.pull_requests'].search([], order='number')
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index a9f6a3d1..8e9a3674 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -20,6 +20,11 @@ class Batch(models.Model):
 
     prs = fields.One2many('runbot_merge.pull_requests', 'batch_id')
 
+    fw_policy = fields.Selection([
+        ('default', "Default"),
+        ('skipci', "Skip CI"),
+    ], required=True, default="default", string="Forward Port Policy")
+
     merge_date = fields.Datetime(tracking=True)
     skipchecks = fields.Boolean(
         string="Skips Checks",
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index eb28fe20..74f7dbe7 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -798,7 +798,7 @@ class PullRequests(models.Model):
                     feedback(close=True)
                 case commands.FW():
                     if source_reviewer or is_reviewer:
-                        (self.source_id or self).fw_policy = command.name.lower()
+                        (self.source_id or self).batch_id.fw_policy = command.name.lower()
                         match command:
                             case commands.FW.DEFAULT:
                                 message = "Waiting for CI to create followup forward-ports."
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index dc219101..f4a65b61 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -159,7 +159,7 @@
                             <field name="squash"/>
                             <field name="draft"/>
                             <field string="Up To" name="limit_id"/>
-                            <field string="Forward-Port Policy" name="fw_policy"/>
+<!--                            <field string="Forward-Port Policy" name="fw_policy"/>-->
                         </group>
                     </group>
                     <!-- influencers -->

From ef6a002ea7b387c75f00759858ea16e61287030e Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 5 Feb 2024 16:10:15 +0100
Subject: [PATCH 086/192] [CHG] runbot_merge: move staging readiness to batch

Staging readiness is a batch-level concerns, and many of the markers
are already there though a few need to be aggregated from the PRs. As
such, staging has no reason to be performed in terms of PRs anymore,
it should be performed via batches directly.

There is a bit of a mess in order not to completely fuck up when
retargeting PRs (implicitly via freeze wizard, or explicitely) as for
now we're moving PRs between batches in order to keep the
batches *mostly* target-bound.

Some of the side-effects in managing the coherence of the targeting
and moving PRs between batches is... not great. This might need to be
revisited and cleaned up with those scenarios better considered.
---
 runbot_merge/controllers/__init__.py          |   2 +-
 runbot_merge/models/batch.py                  |  28 ++++-
 .../models/project_freeze/__init__.py         |  13 +-
 runbot_merge/models/pull_requests.py          |  53 ++++----
 runbot_merge/models/stagings_create.py        | 114 ++++++++----------
 runbot_merge/tests/test_basic.py              |   1 +
 6 files changed, 118 insertions(+), 93 deletions(-)

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 006f441e..5485f1fa 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -176,7 +176,7 @@ def handle_pr(env, event):
         return env['runbot_merge.pull_requests'].search([
             ('repository', '=', repo.id),
             ('number', '=', pr['number']),
-            ('target', '=', target.id),
+            # ('target', '=', target.id),
         ])
     # edition difficulty: pr['base']['ref] is the *new* target, the old one
     # is at event['change']['base']['ref'] (if the target changed), so edition
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index 8e9a3674..bb49c857 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from odoo import models, fields
+from odoo import models, fields, api
 
 
 class Batch(models.Model):
@@ -36,3 +36,29 @@ class Batch(models.Model):
         default=False, tracking=True,
         help="Cancels current staging on target branch when becoming ready"
     )
+
+    blocked = fields.Char(store=True, compute="_compute_stageable")
+
+    @api.depends(
+        "merge_date",
+        "prs.error", "prs.draft", "prs.squash", "prs.merge_method",
+        "skipchecks", "prs.status", "prs.reviewed_by",
+    )
+    def _compute_stageable(self):
+        for batch in self:
+            if batch.merge_date:
+                batch.blocked = "Merged."
+            elif blocking := batch.prs.filtered(
+                lambda p: p.error or p.draft or not (p.squash or p.merge_method)
+            ):
+                batch.blocked = "Pull request(s) %s blocked." % (
+                    p.display_name for p in blocking
+                )
+            elif not batch.skipchecks and (unready := batch.prs.filtered(
+                lambda p: not (p.reviewed_by and p.status == "success")
+            )):
+                batch.blocked = "Pull request(s) %s waiting for CI." % ', '.join(
+                    p.display_name for p in unready
+                )
+            else:
+                batch.blocked = False
diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py
index 7215d762..4114c944 100644
--- a/runbot_merge/models/project_freeze/__init__.py
+++ b/runbot_merge/models/project_freeze/__init__.py
@@ -218,7 +218,8 @@ class FreezeWizard(models.Model):
         }
         for repo, copy in repos.items():
             copy.fetch(git.source_url(repo, 'github'), '+refs/heads/*:refs/heads/*')
-        for pr in self.release_pr_ids.pr_id | self.bump_pr_ids.pr_id:
+        all_prs = self.release_pr_ids.pr_id | self.bump_pr_ids.pr_id
+        for pr in all_prs:
             repos[pr.repository].fetch(
                 git.source_url(pr.repository, 'github'),
                 pr.head,
@@ -338,7 +339,15 @@ class FreezeWizard(models.Model):
                 f"Unable to {reason} branch {repo}:{branch}.{addendum}"
             )
 
-        all_prs = self.release_pr_ids.pr_id | self.bump_pr_ids.pr_id
+        b = self.env['runbot_merge.branch'].search([('name', '=', self.branch_name)])
+        self.env.cr.execute(
+            "UPDATE runbot_merge_batch SET target=%s WHERE id = %s;"
+            "UPDATE runbot_merge_pull_requests SET target=%s WHERE id = any(%s)",
+            [
+                b.id, self.release_pr_ids.pr_id.batch_id.id,
+                b.id, self.release_pr_ids.pr_id.ids,
+            ]
+        )
         all_prs.batch_id.merge_date = fields.Datetime.now()
         all_prs.reviewed_by = self.env.user.partner_id.id
         self.env['runbot_merge.pull_requests.feedback'].create([{
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 74f7dbe7..931ade1c 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1094,15 +1094,12 @@ class PullRequests(models.Model):
         if canceler:
             fields.append('state')
             fields.append('skipchecks')
-        if 'target' in vals:
-            fields.append('target')
-        if 'message' in vals:
-            fields.append('message')
         prev = {pr.id: {field: pr[field] for field in fields} for pr in self}
 
+        # when explicitly marking a PR as ready
         if vals.get('state') == 'ready':
             # skip checks anyway
-            vals['skipchecks'] = True
+            self.batch_id.skipchecks = True
             # if the state is forced to ready, set current user as reviewer
             # and override all statuses
             vals.setdefault('reviewed_by', self.env.user.partner_id.id)
@@ -1117,6 +1114,20 @@ class PullRequests(models.Model):
                     ('prs', '=', True),
                 ])
             }))
+
+        for pr in self:
+            if (t := vals.get('target')) is not None and pr.target.id != t:
+                pr.unstage(
+                    "target (base) branch was changed from %r to %r",
+                    pr.target.display_name,
+                    self.env['runbot_merge.branch'].browse(t).display_name,
+                )
+
+            if 'message' in vals:
+                merge_method = vals['merge_method'] if 'merge_method' in vals else pr.merge_method
+                if merge_method not in (False, 'rebase-ff') and pr.message != vals['message']:
+                    pr.unstage("merge message updated")
+
         match vals.get('closed'):
             case True if not self.closed:
                 vals['reviewed_by'] = False
@@ -1128,6 +1139,13 @@ class PullRequests(models.Model):
                     target=vals.get('target') or self.target.id,
                     label=vals.get('label') or self.label,
                 )
+            case None if (new_target := vals.get('target')) and new_target != self.target.id:
+                vals['batch_id'] = self._get_batch(
+                    target=new_target,
+                    label=vals.get('label') or self.label,
+                )
+                if len(self.batch_id.prs) == 1:
+                    self.env.cr.precommit.add(self.batch_id.unlink)
         w = super().write(vals)
 
         newhead = vals.get('head')
@@ -1135,29 +1153,20 @@ class PullRequests(models.Model):
             c = self.env['runbot_merge.commit'].search([('sha', '=', newhead)])
             self._validate(c.statuses or '{}')
 
-        for pr in self:
-            old = prev[pr.id]
-            if canceler:
+        if canceler:
+            for pr in self:
+                if not pr.cancel_staging:
+                    continue
+
+                old = prev[pr.id]
                 def ready(pr):
                     return pr['state'] == 'ready'\
                        or (pr['skipchecks'] and pr['state'] != 'error')
-                if pr.cancel_staging and not ready(old) and ready(pr):
-                    if old['state'] == 'error': # error -> ready gets a bespok message
+                if ready(pr) and not ready(old):
+                    if old['state'] == 'error': # error -> ready gets a bespoke message
                         pr.target.active_staging_id.cancel(f"retrying {pr.display_name}")
                     else:
                         pr.target.active_staging_id.cancel(f"{pr.display_name} became ready")
-
-            if 'target' in vals:
-                old_target = old['target']
-                if pr.target != old_target:
-                    pr.unstage(
-                        "target (base) branch was changed from %r to %r",
-                        old_target.display_name, pr.target.display_name,
-                    )
-
-            if 'message' in vals:
-                if pr.merge_method not in (False, 'rebase-ff') and pr.message != old['message']:
-                    pr.unstage("merge message updated")
         return w
 
     def _check_linked_prs_statuses(self, commit=False):
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index f7f6f971..fc893937 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -6,6 +6,7 @@ import json
 import logging
 import os
 import re
+from collections.abc import Mapping
 from difflib import Differ
 from itertools import takewhile
 from operator import itemgetter
@@ -14,7 +15,7 @@ from typing import Dict, Union, Optional, Literal, Callable, Iterator, Tuple, Li
 from werkzeug.datastructures import Headers
 
 from odoo import api, models, fields
-from odoo.tools import OrderedSet
+from odoo.tools import OrderedSet, groupby
 from .pull_requests import Branch, Stagings, PullRequests, Repository
 from .batch import Batch
 from .. import exceptions, utils, github, git
@@ -57,76 +58,63 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
     if branch.active_staging_id:
         return None
 
-    rows = [
-        (p, prs)
-        for p, prs in ready_prs(for_branch=branch)
-        if not any(prs.mapped('blocked'))
-    ]
-
-    def log(label, batches):
-        _logger.info(label, ', '.join(
-            p.display_name
-            for batch in batches
-            for p in batch
-        ))
+    def log(label: str, batches: Batch) -> None:
+        _logger.info(label, ', '.join(batches.mapped('prs.display_name')))
 
+    rows = ready_prs(for_branch=branch)
     priority = rows[0][0] if rows else None
+    concat = branch.env['runbot_merge.batch'].concat
     if priority == 'alone':
-        batched_prs = [pr_ids for _, pr_ids in takewhile(lambda r: r[0] == priority, rows)]
-        log("staging high-priority PRs %s", batched_prs)
+        batches: Batch = concat(*(batch for _, batch in takewhile(lambda r: r[0] == priority, rows)))
+        log("staging high-priority PRs %s", batches)
     elif branch.project_id.staging_priority == 'default':
-        if branch.split_ids:
-            split_ids = branch.split_ids[0]
-            batched_prs = [batch.prs for batch in split_ids.batch_ids]
-            split_ids.unlink()
-            log("staging split PRs %s (prioritising splits)", batched_prs)
+        if split := branch.split_ids[:1]:
+            batches = split.batch_ids
+            split.unlink()
+            log("staging split PRs %s (prioritising splits)", batches)
         else:
             # priority, normal; priority = sorted ahead of normal, so always picked
             # first as long as there's room
-            batched_prs = [pr_ids for _, pr_ids in rows]
-            log("staging ready PRs %s (prioritising splits)", batched_prs)
+            batches = concat(*(batch for _, batch in rows))
+            log("staging ready PRs %s (prioritising splits)", batches)
     elif branch.project_id.staging_priority == 'ready':
         # splits are ready by definition, we need to exclude them from the
         # ready rows otherwise we'll re-stage the splits so if an error is legit
         # we cycle forever
         # FIXME: once the batches are less shit, store this durably on the
         #        batches and filter out when fetching readies (?)
-        split_batches = {batch.prs for batch in branch.split_ids.batch_ids}
-        ready = [pr_ids for _, pr_ids in rows if pr_ids not in split_batches]
+        batches = concat(*(batch for _, batch in rows)) - branch.split_ids.batch_ids
 
-        if ready:
-            batched_prs = ready
-            log("staging ready PRs %s (prioritising ready)", batched_prs)
+        if batches:
+            log("staging ready PRs %s (prioritising ready)", batches)
         else:
-            split_ids = branch.split_ids[:1]
-            batched_prs = [batch.prs for batch in split_ids.batch_ids]
-            split_ids.unlink()
-            log("staging split PRs %s (prioritising ready)", batched_prs)
+            split = branch.split_ids[:1]
+            batches = split.batch_ids
+            split.unlink()
+            log("staging split PRs %s (prioritising ready)", batches)
     else:
         assert branch.project_id.staging_priority == 'largest'
         # splits are ready by definition, we need to exclude them from the
         # ready rows otherwise ready always wins but we re-stage the splits, so
         # if an error is legit we'll cycle forever
-        split_batches = {batch.prs for batch in branch.split_ids.batch_ids}
-        ready = [pr_ids for _, pr_ids in rows if pr_ids not in split_batches]
+        batches = concat(*(batch for _, batch in rows)) - branch.split_ids.batch_ids
 
         maxsplit = max(branch.split_ids, key=lambda s: len(s.batch_ids), default=branch.env['runbot_merge.split'])
-        _logger.info("largest split = %d, ready = %d", len(maxsplit.batch_ids), len(ready))
+        _logger.info("largest split = %d, ready = %d", len(maxsplit.batch_ids), len(batches))
         # bias towards splits if len(ready) = len(batch_ids)
-        if len(maxsplit.batch_ids) >= len(ready):
-            batched_prs = [batch.prs for batch in maxsplit.batch_ids]
+        if len(maxsplit.batch_ids) >= len(batches):
+            batches = maxsplit.batch_ids
             maxsplit.unlink()
-            log("staging split PRs %s (prioritising largest)", batched_prs)
+            log("staging split PRs %s (prioritising largest)", batches)
         else:
-            batched_prs = ready
-            log("staging ready PRs %s (prioritising largest)", batched_prs)
+            log("staging ready PRs %s (prioritising largest)", batches)
 
-    if not batched_prs:
+    if not batches:
         return
 
-    original_heads, staging_state = staging_setup(branch, batched_prs)
+    original_heads, staging_state = staging_setup(branch, batches)
 
-    staged = stage_batches(branch, batched_prs, staging_state)
+    staged = stage_batches(branch, batches, staging_state)
 
     if not staged:
         return None
@@ -216,32 +204,25 @@ For-Commit-Id: {it.head}
     return st
 
 
-def ready_prs(for_branch: Branch) -> List[Tuple[int, PullRequests]]:
+def ready_prs(for_branch: Branch) -> List[Tuple[int, Batch]]:
     env = for_branch.env
     env.cr.execute("""
     SELECT
       max(pr.priority) as priority,
-      array_agg(pr.id) AS match
-    FROM runbot_merge_pull_requests pr
-    JOIN runbot_merge_batch b ON (b.id = pr.batch_id)
-    WHERE pr.target = any(%s)
-      -- exclude terminal states (so there's no issue when
-      -- deleting branches & reusing labels)
-      AND pr.state != 'merged'
-      AND pr.state != 'closed'
+      b.id as batch
+    FROM runbot_merge_batch b
+    JOIN runbot_merge_pull_requests pr ON (b.id = pr.batch_id)
+    WHERE b.target = %s AND b.blocked IS NULL
     GROUP BY b.id
-    HAVING
-        bool_and(pr.state = 'ready')
-     OR (bool_or(b.skipchecks) AND bool_and(pr.state != 'error'))
-    ORDER BY max(pr.priority) DESC, min(pr.id)
-    """, [for_branch.ids])
-    browse = env['runbot_merge.pull_requests'].browse
-    return [(p, browse(ids)) for p, ids in env.cr.fetchall()]
+    ORDER BY max(pr.priority) DESC, min(b.id)
+    """, [for_branch.id])
+    browse = env['runbot_merge.batch'].browse
+    return [(p, browse(id)) for p, id in env.cr.fetchall()]
 
 
 def staging_setup(
         target: Branch,
-        batched_prs: List[PullRequests],
+        batches: Batch,
 ) -> Tuple[Dict[Repository, str], StagingState]:
     """Sets up the staging:
 
@@ -249,7 +230,9 @@ def staging_setup(
     - creates tmp branch via gh API (to remove)
     - generates working copy for each repository with the target branch
     """
-    all_prs: PullRequests = target.env['runbot_merge.pull_requests'].concat(*batched_prs)
+    by_repo: Mapping[Repository, List[PullRequests]] = \
+        dict(groupby(batches.prs, lambda p: p.repository))
+
     staging_state = {}
     original_heads = {}
     for repo in target.project_id.repo_ids.having_branch(target):
@@ -265,7 +248,7 @@ def staging_setup(
             # be hooked only to "proper" remote-tracking branches
             # (in `refs/remotes`), it doesn't seem to work here
             f'+refs/heads/{target.name}:refs/heads/{target.name}',
-            *(pr.head for pr in all_prs if pr.repository == repo)
+            *(pr.head for pr in by_repo.get(repo, []))
         )
         original_heads[repo] = head
         staging_state[repo] = StagingSlice(gh=gh, head=head, repo=source.stdout().with_config(text=True, check=False))
@@ -273,18 +256,15 @@ def staging_setup(
     return original_heads, staging_state
 
 
-def stage_batches(branch: Branch, batched_prs: List[PullRequests], staging_state: StagingState) -> Stagings:
+def stage_batches(branch: Branch, batches: Batch, staging_state: StagingState) -> Stagings:
     batch_limit = branch.project_id.batch_limit
     env = branch.env
     staged = env['runbot_merge.batch']
-    for batch in batched_prs:
+    for batch in batches:
         if len(staged) >= batch_limit:
             break
-
-        assert len(bs := {p.batch_id for p in batch}) == 1,\
-            f"expected all PRs to have the same batch, got {bs}"
         try:
-            staged |= stage_batch(env, batch[0].batch_id, staging_state)
+            staged |= stage_batch(env, batch, staging_state)
         except exceptions.MergeError as e:
             pr = e.args[0]
             _logger.info("Failed to stage %s into %s", pr.display_name, branch.name, exc_info=True)
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 2d0bc440..4ca95c60 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -773,6 +773,7 @@ class TestPREdition:
         with repo: prx.base = '1.0'
         assert pr.target == branch_1
         assert not pr.staging_id, "updated the base of a staged PR should have unstaged it"
+        assert st.state == 'cancelled', f"expected cancellation, got {st.state}"
         assert st.reason == f"{pr.display_name} target (base) branch was changed from 'master' to '1.0'"
 
         with repo: prx.base = '2.0'

From 83511f45e2666ce59781b7f84148dc32b7c65c7a Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 7 Feb 2024 15:05:33 +0100
Subject: [PATCH 087/192] [CHG] runbot_merge: move priority field from PR to
 batch

Simplifies the `ready_prs` query a bit and allows it to be converted
to an ORM search, by moving the priority check outside. This also
allows the caller to not need to post-process the records list
anywhere near the previous state of affairs.

`ready_prs` now returns *either* the "alone" batches, or the non-alone
batches, rather than mixing both into a single sequence. This requires
correctly applying the search filters to not retrieve priority of
batches in error or targeting other branches.
---
 runbot_merge/models/batch.py           | 31 +++++++++++++++++++++
 runbot_merge/models/pull_requests.py   | 18 ++++--------
 runbot_merge/models/stagings_create.py | 38 +++++++++++++-------------
 runbot_merge/models/utils.py           |  6 ++++
 runbot_merge/tests/test_basic.py       |  6 ++++
 5 files changed, 67 insertions(+), 32 deletions(-)
 create mode 100644 runbot_merge/models/utils.py

diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index bb49c857..d3d2aacd 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 from odoo import models, fields, api
+from odoo.tools import create_index
+from .utils import enum
 
 
 class Batch(models.Model):
@@ -36,9 +38,38 @@ class Batch(models.Model):
         default=False, tracking=True,
         help="Cancels current staging on target branch when becoming ready"
     )
+    priority = fields.Selection([
+        ('default', "Default"),
+        ('priority', "Priority"),
+        ('alone', "Alone"),
+    ], default='default', index=True, group_operator=None, required=True,
+        column_type=enum(_name, 'priority'),
+    )
 
     blocked = fields.Char(store=True, compute="_compute_stageable")
 
+    def _auto_init(self):
+        for field in self._fields.values():
+            if not isinstance(field, fields.Selection) or field.column_type[0] == 'varchar':
+                continue
+
+            t = field.column_type[1]
+            self.env.cr.execute("SELECT FROM pg_type WHERE typname = %s", [t])
+            if not self.env.cr.rowcount:
+                self.env.cr.execute(
+                    f"CREATE TYPE {t} AS ENUM %s",
+                    [tuple(s for s, _ in field.selection)]
+                )
+
+        super()._auto_init()
+
+        create_index(
+            self.env.cr,
+            "runbot_merge_batch_unblocked_idx",
+            "runbot_merge_batch",
+            ["(blocked is null), priority"],
+        )
+
     @api.depends(
         "merge_date",
         "prs.error", "prs.draft", "prs.squash", "prs.merge_method",
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 931ade1c..fb34361a 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -10,7 +10,7 @@ import logging
 import re
 import time
 from functools import reduce
-from typing import Optional, Union, List, Iterator, Tuple
+from typing import Optional, Union, List, Iterator
 
 import sentry_sdk
 import werkzeug
@@ -19,6 +19,7 @@ from odoo import api, fields, models, tools, Command
 from odoo.osv import expression
 from odoo.tools import html_escape
 from . import commands
+from .utils import enum
 
 from .. import github, exceptions, controllers, utils
 
@@ -307,9 +308,6 @@ class Branch(models.Model):
 
 
 ACL = collections.namedtuple('ACL', 'is_admin is_reviewer is_author')
-def enum(model: str, field: str) -> Tuple[str, str]:
-    n = f'{model.replace(".", "_")}_{field}_type'
-    return n, n
 class PullRequests(models.Model):
     _name = 'runbot_merge.pull_requests'
     _description = "Pull Request"
@@ -374,13 +372,7 @@ class PullRequests(models.Model):
 
     reviewed_by = fields.Many2one('res.partner', index=True, tracking=True)
     delegates = fields.Many2many('res.partner', help="Delegate reviewers, not intrinsically reviewers but can review this PR")
-    priority = fields.Selection([
-        ('default', "Default"),
-        ('priority', "Priority"),
-        ('alone', "Alone"),
-    ], default='default', index=True, group_operator=None, required=True,
-        column_type=enum(_name, 'priority'),
-    )
+    priority = fields.Selection(related="batch_id.priority", inverse=lambda _: 1 / 0)
 
     overrides = fields.Char(required=True, default='{}', tracking=True)
     statuses = fields.Text(help="Copy of the statuses from the HEAD commit, as a Python literal", default="{}")
@@ -755,7 +747,7 @@ class PullRequests(models.Model):
                             })
                     delegates.write({'delegate_reviewer': [(4, self.id, 0)]})
                 case commands.Priority() if is_admin:
-                    self.priority = str(command)
+                    self.batch_id.priority = str(command)
                 case commands.SkipChecks() if is_admin:
                     self.batch_id.skipchecks = True
                     self.reviewed_by = author
@@ -1014,7 +1006,7 @@ class PullRequests(models.Model):
                     [tuple(s for s, _ in field.selection)]
                 )
 
-        super(PullRequests, self)._auto_init()
+        super()._auto_init()
         # incorrect index: unique(number, target, repository).
         tools.drop_index(self._cr, 'runbot_merge_unique_pr_per_target', self._table)
         # correct index:
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index fc893937..231ee969 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -61,11 +61,9 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
     def log(label: str, batches: Batch) -> None:
         _logger.info(label, ', '.join(batches.mapped('prs.display_name')))
 
-    rows = ready_prs(for_branch=branch)
-    priority = rows[0][0] if rows else None
-    concat = branch.env['runbot_merge.batch'].concat
-    if priority == 'alone':
-        batches: Batch = concat(*(batch for _, batch in takewhile(lambda r: r[0] == priority, rows)))
+    alone, batches = ready_prs(for_branch=branch)
+    print('alone', alone, 'ready', batches)
+    if alone:
         log("staging high-priority PRs %s", batches)
     elif branch.project_id.staging_priority == 'default':
         if split := branch.split_ids[:1]:
@@ -75,7 +73,6 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
         else:
             # priority, normal; priority = sorted ahead of normal, so always picked
             # first as long as there's room
-            batches = concat(*(batch for _, batch in rows))
             log("staging ready PRs %s (prioritising splits)", batches)
     elif branch.project_id.staging_priority == 'ready':
         # splits are ready by definition, we need to exclude them from the
@@ -83,7 +80,7 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
         # we cycle forever
         # FIXME: once the batches are less shit, store this durably on the
         #        batches and filter out when fetching readies (?)
-        batches = concat(*(batch for _, batch in rows)) - branch.split_ids.batch_ids
+        batches -= branch.split_ids.batch_ids
 
         if batches:
             log("staging ready PRs %s (prioritising ready)", batches)
@@ -97,7 +94,7 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
         # splits are ready by definition, we need to exclude them from the
         # ready rows otherwise ready always wins but we re-stage the splits, so
         # if an error is legit we'll cycle forever
-        batches = concat(*(batch for _, batch in rows)) - branch.split_ids.batch_ids
+        batches -= branch.split_ids.batch_ids
 
         maxsplit = max(branch.split_ids, key=lambda s: len(s.batch_ids), default=branch.env['runbot_merge.split'])
         _logger.info("largest split = %d, ready = %d", len(maxsplit.batch_ids), len(batches))
@@ -204,20 +201,23 @@ For-Commit-Id: {it.head}
     return st
 
 
-def ready_prs(for_branch: Branch) -> List[Tuple[int, Batch]]:
+def ready_prs(for_branch: Branch) -> Tuple[bool, Batch]:
     env = for_branch.env
     env.cr.execute("""
-    SELECT
-      max(pr.priority) as priority,
-      b.id as batch
-    FROM runbot_merge_batch b
-    JOIN runbot_merge_pull_requests pr ON (b.id = pr.batch_id)
-    WHERE b.target = %s AND b.blocked IS NULL
-    GROUP BY b.id
-    ORDER BY max(pr.priority) DESC, min(b.id)
+        SELECT max(priority)
+        FROM runbot_merge_batch
+        WHERE blocked IS NULL AND target = %s
     """, [for_branch.id])
-    browse = env['runbot_merge.batch'].browse
-    return [(p, browse(id)) for p, id in env.cr.fetchall()]
+    alone = env.cr.fetchone()[0] == 'alone'
+
+    return (
+        alone,
+        env['runbot_merge.batch'].search([
+            ('target', '=', for_branch.id),
+            ('blocked', '=', False),
+            ('priority', '=', 'alone') if alone else (1, '=', 1),
+        ], order="priority DESC, id ASC"),
+    )
 
 
 def staging_setup(
diff --git a/runbot_merge/models/utils.py b/runbot_merge/models/utils.py
new file mode 100644
index 00000000..6aeeb9b8
--- /dev/null
+++ b/runbot_merge/models/utils.py
@@ -0,0 +1,6 @@
+from typing import Tuple
+
+
+def enum(model: str, field: str) -> Tuple[str, str]:
+    n = f'{model.replace(".", "_")}_{field}_type'
+    return n, n
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 4ca95c60..273c0aae 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -2820,11 +2820,17 @@ class TestBatching(object):
         assert not staging_1.active
         assert not p_11.staging_id and not p_12.staging_id
         assert p_01.staging_id
+        assert p_11.state == 'ready'
+        assert p_12.state == 'ready'
 
         # make the staging fail
         with repo:
             repo.post_status('staging.master', 'failure', 'ci/runbot')
         env.run_crons()
+        assert p_01.error
+        assert p_01.batch_id.blocked
+        assert p_01.blocked
+
         assert p_01.state == 'error'
         assert not p_01.staging_id.active
         staging_2 = ensure_one(sm_all.staging_id)

From ad1d590d9ca8c87c58bc1cf5c9c945dbad3cbdd3 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 8 Feb 2024 12:02:34 +0100
Subject: [PATCH 088/192] [IMP] runbot_merge: fix dual merge of split
 prioritised PRs

Because `alone` (formerly p != 2) is selected before split PRs, if a
prioritised PR gets split (or a split PR gets prioritised) it will be
staged once as prioritised, and again because split.

Improve the selection of ready batches to exclude split batches
upstream, such that they don't have to be rechecked over and over, and
their priorities don't cause us issues.
---
 runbot_merge/models/stagings_create.py | 27 ++++++----------
 runbot_merge/tests/test_basic.py       | 44 ++++++++++++++++++++++++++
 2 files changed, 54 insertions(+), 17 deletions(-)

diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index 231ee969..017b63a9 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -61,8 +61,8 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
     def log(label: str, batches: Batch) -> None:
         _logger.info(label, ', '.join(batches.mapped('prs.display_name')))
 
-    alone, batches = ready_prs(for_branch=branch)
-    print('alone', alone, 'ready', batches)
+    alone, batches = ready_batches(for_branch=branch)
+
     if alone:
         log("staging high-priority PRs %s", batches)
     elif branch.project_id.staging_priority == 'default':
@@ -75,13 +75,6 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
             # first as long as there's room
             log("staging ready PRs %s (prioritising splits)", batches)
     elif branch.project_id.staging_priority == 'ready':
-        # splits are ready by definition, we need to exclude them from the
-        # ready rows otherwise we'll re-stage the splits so if an error is legit
-        # we cycle forever
-        # FIXME: once the batches are less shit, store this durably on the
-        #        batches and filter out when fetching readies (?)
-        batches -= branch.split_ids.batch_ids
-
         if batches:
             log("staging ready PRs %s (prioritising ready)", batches)
         else:
@@ -91,11 +84,6 @@ def try_staging(branch: Branch) -> Optional[Stagings]:
             log("staging split PRs %s (prioritising ready)", batches)
     else:
         assert branch.project_id.staging_priority == 'largest'
-        # splits are ready by definition, we need to exclude them from the
-        # ready rows otherwise ready always wins but we re-stage the splits, so
-        # if an error is legit we'll cycle forever
-        batches -= branch.split_ids.batch_ids
-
         maxsplit = max(branch.split_ids, key=lambda s: len(s.batch_ids), default=branch.env['runbot_merge.split'])
         _logger.info("largest split = %d, ready = %d", len(maxsplit.batch_ids), len(batches))
         # bias towards splits if len(ready) = len(batch_ids)
@@ -201,13 +189,17 @@ For-Commit-Id: {it.head}
     return st
 
 
-def ready_prs(for_branch: Branch) -> Tuple[bool, Batch]:
+def ready_batches(for_branch: Branch) -> Tuple[bool, Batch]:
     env = for_branch.env
+    # splits are ready by definition, we need to exclude them from the ready
+    # rows otherwise if a prioritised (alone) PR is part of a split it'll be
+    # staged through priority *and* through split.
+    split_ids = for_branch.split_ids.batch_ids.ids
     env.cr.execute("""
         SELECT max(priority)
         FROM runbot_merge_batch
-        WHERE blocked IS NULL AND target = %s
-    """, [for_branch.id])
+        WHERE blocked IS NULL AND target = %s AND NOT id = any(%s)
+    """, [for_branch.id, split_ids])
     alone = env.cr.fetchone()[0] == 'alone'
 
     return (
@@ -216,6 +208,7 @@ def ready_prs(for_branch: Branch) -> Tuple[bool, Batch]:
             ('target', '=', for_branch.id),
             ('blocked', '=', False),
             ('priority', '=', 'alone') if alone else (1, '=', 1),
+            ('id', 'not in', split_ids),
         ], order="priority DESC, id ASC"),
     )
 
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 273c0aae..dc458b34 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -2969,6 +2969,50 @@ class TestBatching(object):
         assert not p_01.staging_id, "p_01 should not be picked up as it's failed"
         assert p_21.staging_id, "p_21 should have been staged"
 
+    def test_urgent_split(self, env, repo, config):
+        """Ensure that urgent (alone) PRs which get split don't get
+        double-merged
+        """
+        with repo:
+            repo.make_commits(
+                None,
+                Commit("initial", tree={'a': '1'}),
+                ref="heads/master"
+            )
+
+            pr01 = self._pr(
+                repo, "PR1", [{'b': '1'}],
+                user=config['role_user']['token'],
+                reviewer=None,
+            )
+            pr01.post_comment('hansen alone r+', config['role_reviewer']['token'])
+            pr02 = self._pr(
+                repo, "PR2", [{'c': '1'}],
+                user=config['role_user']['token'],
+                reviewer=None,
+            )
+            pr02.post_comment('hansen alone r+', config['role_reviewer']['token'])
+        env.run_crons('runbot_merge.process_updated_commits')
+        pr01_id = to_pr(env, pr01)
+        assert pr01_id.blocked is False
+        pr02_id = to_pr(env, pr02)
+        assert pr01_id.blocked is False
+
+        env.run_crons()
+        st = pr01_id.staging_id
+        assert st and pr02_id.staging_id == st
+        with repo:
+            repo.post_status('staging.master', 'failure', 'ci/runbot')
+        env.run_crons()
+        # should have cancelled the staging, split it, and re-staged the first
+        # half of the split
+        assert st.state == 'failure'
+        assert pr01_id.staging_id and pr01_id.staging_id != st
+        assert not pr02_id.staging_id
+        split_prs = env['runbot_merge.split'].search([]).batch_ids.prs
+        assert split_prs == pr02_id, \
+            f"only the unstaged PR {pr02_id} should be in a split, found {split_prs}"
+
     @pytest.mark.skip(reason="Maybe nothing to do, the PR is just skipped and put in error?")
     def test_batching_merge_failure(self):
         pass

From a6a37f8896ae3c0728ec1e83f5edb9dbb7f0f350 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 9 Feb 2024 08:58:19 +0100
Subject: [PATCH 089/192] [FIX] runbot_merge: handling of staging cancellation

Move staging cancellation to the batch, remove its (complicated)
handling from the PRs.

This loses some precision in the cancellation message, but that could
likely be recovered (in part) by adding more precise checks &
diagnostic extractions in the compute.
---
 conftest.py                          |  3 ++
 runbot_merge/models/batch.py         |  7 ++++
 runbot_merge/models/commands.py      |  2 +
 runbot_merge/models/pull_requests.py | 36 +++-------------
 runbot_merge/tests/test_multirepo.py | 63 ++++++++++++++++++++++++++++
 5 files changed, 81 insertions(+), 30 deletions(-)

diff --git a/conftest.py b/conftest.py
index 2bf6f4e0..db3822f7 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1175,6 +1175,9 @@ class Model:
     def __len__(self):
         return len(self._ids)
 
+    def __hash__(self):
+        return hash((self._model, frozenset(self._ids)))
+
     def __eq__(self, other):
         if not isinstance(other, Model):
             return NotImplemented
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index d3d2aacd..c7b82b30 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -92,4 +92,11 @@ class Batch(models.Model):
                     p.display_name for p in unready
                 )
             else:
+                if batch.blocked and batch.cancel_staging:
+                    batch.target.active_staging_id.cancel(
+                        'unstaged by %s on %s (%s)',
+                        self.env.user.login,
+                        batch,
+                        ', '.join(batch.prs.mapped('display_name')),
+                    )
                 batch.blocked = False
diff --git a/runbot_merge/models/commands.py b/runbot_merge/models/commands.py
index 7d097d47..0f46251f 100644
--- a/runbot_merge/models/commands.py
+++ b/runbot_merge/models/commands.py
@@ -281,6 +281,8 @@ class Parser:
         return Priority.ALONE
 
     def parse_cancel(self) -> CancelStaging:
+        self.assert_next('=')
+        self.assert_next('staging')
         return CancelStaging()
 
     def parse_skipchecks(self) -> SkipChecks:
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index fb34361a..02eab8c4 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -700,7 +700,7 @@ class PullRequests(models.Model):
                     else:
                         msg = self._approve(author, login)
                 case commands.Reject() if is_author:
-                    if cancellers := self.batch_id.cancel_staging:
+                    if self.batch_id.cancel_staging:
                         self.batch_id.cancel_staging = False
                     if self.batch_id.skipchecks or self.reviewed_by:
                         if self.error:
@@ -756,14 +756,11 @@ class PullRequests(models.Model):
                             p.reviewed_by = author
                 case commands.CancelStaging() if is_admin:
                     self.batch_id.cancel_staging = True
-                    # FIXME: remove this when skipchecks properly affects state,
-                    #        maybe: staging cancellation should then only occur
-                    #        when a cancel_staging PR transitions to ready, or
-                    #        a ready PR is flagged as cancelling staging
-                    self.target.active_staging_id.cancel(
-                        "Unstaged by %s on %s",
-                        author.github_login, self.display_name,
-                    )
+                    if not self.batch_id.blocked:
+                        self.target.active_staging_id.cancel(
+                            "Unstaged by %s on %s",
+                            author.github_login, self.display_name,
+                        )
                 case commands.Override(statuses):
                     for status in statuses:
                         overridable = author.override_rights\
@@ -1081,12 +1078,6 @@ class PullRequests(models.Model):
     def write(self, vals):
         if vals.get('squash'):
             vals['merge_method'] = False
-        fields = []
-        canceler = vals.get('cancel_staging') or any(p.cancel_staging for p in self)
-        if canceler:
-            fields.append('state')
-            fields.append('skipchecks')
-        prev = {pr.id: {field: pr[field] for field in fields} for pr in self}
 
         # when explicitly marking a PR as ready
         if vals.get('state') == 'ready':
@@ -1144,21 +1135,6 @@ class PullRequests(models.Model):
         if newhead:
             c = self.env['runbot_merge.commit'].search([('sha', '=', newhead)])
             self._validate(c.statuses or '{}')
-
-        if canceler:
-            for pr in self:
-                if not pr.cancel_staging:
-                    continue
-
-                old = prev[pr.id]
-                def ready(pr):
-                    return pr['state'] == 'ready'\
-                       or (pr['skipchecks'] and pr['state'] != 'error')
-                if ready(pr) and not ready(old):
-                    if old['state'] == 'error': # error -> ready gets a bespoke message
-                        pr.target.active_staging_id.cancel(f"retrying {pr.display_name}")
-                    else:
-                        pr.target.active_staging_id.cancel(f"{pr.display_name} became ready")
         return w
 
     def _check_linked_prs_statuses(self, commit=False):
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 3c72b392..813a2685 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -9,6 +9,7 @@ import functools
 import operator
 import time
 import xmlrpc.client
+from itertools import repeat
 
 import pytest
 import requests
@@ -1468,3 +1469,65 @@ def test_freeze_conflict(env, project, repo_a, repo_b, repo_c, users, config):
     with pytest.raises(AssertionError) as e:
         repo_b.get_ref('heads/1.1')
     assert e.value.args[0].startswith("Not Found")
+
+def test_cancel_staging(env, project, repo_a, repo_b, users, config):
+    """If a batch is flagged as staging cancelling (from any PR), the staging
+    should get cancelled if and when the batch transitions to unblocked
+    """
+    with repo_a, repo_b:
+        make_branch(repo_a, 'master', 'initial', {'a': '1'})
+        make_branch(repo_b, 'master', 'initial', {'b': '1'})
+
+        pr_a = make_pr(repo_a, 'batch', [{'a': '2'}], user=config['role_user']['token'], statuses=[], reviewer=None)
+        pr_b = make_pr(repo_b, 'batch', [{'b': '2'}], user=config['role_user']['token'], statuses=[], reviewer=None)
+        pr_lone = make_pr(
+            repo_a,
+            "C",
+            [{'c': '1'}],
+            user=config['role_user']['token'],
+            reviewer=config['role_reviewer']['token'],
+        )
+    env.run_crons()
+
+    a_id, b_id, lone_id = map(to_pr, repeat(env), [pr_a, pr_b, pr_lone])
+    assert lone_id.staging_id
+    st = lone_id.staging_id
+
+    with repo_a:
+        pr_a.post_comment("hansen cancel=staging", config['role_reviewer']['token'])
+    assert a_id.state == 'opened'
+    assert a_id.cancel_staging
+    assert b_id.cancel_staging
+    assert lone_id.staging_id == st
+    with repo_a:
+        pr_a.post_comment('hansen r+', config['role_reviewer']['token'])
+    assert a_id.state == 'approved'
+    assert lone_id.staging_id == st
+    with repo_a:
+        repo_a.post_status(a_id.head, 'success', 'ci/runbot')
+        repo_a.post_status(a_id.head, 'success', 'legal/cla')
+    env.run_crons()
+    assert a_id.state == 'ready'
+    assert lone_id.staging_id == st
+
+    assert b_id.state == 'opened'
+    with repo_b:
+        pr_b.post_comment('hansen r+', config['role_reviewer']['token'])
+    assert b_id.state == 'approved'
+    assert lone_id.staging_id == st
+    with repo_b:
+        repo_b.post_status(b_id.head, 'success', 'ci/runbot')
+        repo_b.post_status(b_id.head, 'success', 'legal/cla')
+    assert b_id.state == 'approved'
+    assert lone_id.staging_id == st
+    env.run_crons()
+    assert b_id.state == 'ready'
+    # should have cancelled the staging, picked a and b, and re-staged the
+    # entire thing
+    assert lone_id.staging_id != st
+
+    assert len({
+        lone_id.staging_id.id,
+        a_id.staging_id.id,
+        b_id.staging_id.id,
+    }) == 1

From c66451a8c7317c20c336e348707d399390b6cb03 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 9 Feb 2024 10:49:32 +0100
Subject: [PATCH 090/192] [IMP] runbot_merge: cleanup/modernize
 test_multirepo.py

- remove the `legal/cla` and `ci/runbot` context names, which I use a
  lot for historical reasons but fundamentally they're not useful to
  the tests, the `default` context is generally simpler.
- remove `make_branch` helper as we don't actually use branch
  protection and at the end of the day it doesn't do much else
- convert a few explicit PR lookups to the project-wide `to_pr` helper
---
 runbot_merge/tests/test_multirepo.py | 195 +++++++++++----------------
 1 file changed, 79 insertions(+), 116 deletions(-)

diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 813a2685..1f09b817 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -24,7 +24,7 @@ def repo_a(project, make_repo, setreviewers):
     r = project.env['runbot_merge.repository'].create({
         'project_id': project.id,
         'name': repo.name,
-        'required_statuses': 'legal/cla,ci/runbot',
+        'required_statuses': 'default',
         'group_id': False,
     })
     setreviewers(r)
@@ -36,7 +36,7 @@ def repo_b(project, make_repo, setreviewers):
     r = project.env['runbot_merge.repository'].create({
         'project_id': project.id,
         'name': repo.name,
-        'required_statuses': 'legal/cla,ci/runbot',
+        'required_statuses': 'default',
         'group_id': False,
     })
     setreviewers(r)
@@ -48,14 +48,14 @@ def repo_c(project, make_repo, setreviewers):
     r = project.env['runbot_merge.repository'].create({
         'project_id': project.id,
         'name': repo.name,
-        'required_statuses': 'legal/cla,ci/runbot',
+        'required_statuses': 'default',
         'group_id': False,
     })
     setreviewers(r)
     return repo
 
 def make_pr(repo, prefix, trees, *, target='master', user,
-            statuses=(('ci/runbot', 'success'), ('legal/cla', 'success')),
+            statuses=(('default', 'success'),),
             reviewer):
     """
     :type repo: fake_github.Repo
@@ -83,12 +83,6 @@ def make_pr(repo, prefix, trees, *, target='master', user,
         pr.post_comment('hansen r+', reviewer)
     return pr
 
-def make_branch(repo, name, message, tree, protect=True):
-    c = repo.make_commit(None, message, None, tree=tree)
-    repo.make_ref('heads/%s' % name, c)
-    if protect:
-        repo.protect(name)
-    return c
 
 @pytest.mark.parametrize('uniquifier', [False, True])
 def test_stage_one(env, project, repo_a, repo_b, config, uniquifier):
@@ -98,14 +92,14 @@ def test_stage_one(env, project, repo_a, repo_b, config, uniquifier):
     project.batch_limit = 1
 
     with repo_a:
-        make_branch(repo_a, 'master', 'initial', {'a': 'a_0'})
+        repo_a.make_commits(None, Commit('initial', tree={'a': 'a_0'}), ref='heads/master')
         pr_a = make_pr(
             repo_a, 'A', [{'a': 'a_1'}],
             user=config['role_user']['token'],
             reviewer=config['role_reviewer']['token'])
 
     with repo_b:
-        make_branch(repo_b, 'master', 'initial', {'a': 'b_0'})
+        repo_b.make_commits(None, Commit('initial', tree={'a': 'b_0'}), ref='heads/master')
         pr_b = make_pr(
             repo_b, 'B', [{'a': 'b_1'}],
             user=config['role_user']['token'],
@@ -133,14 +127,14 @@ def test_stage_match(env, project, repo_a, repo_b, config, page):
     project.batch_limit = 1
 
     with repo_a:
-        make_branch(repo_a, 'master', 'initial', {'a': 'a_0'})
+        repo_a.make_commits(None, Commit('initial', tree={'a': 'a_0'}), ref='heads/master')
         prx_a = make_pr(
             repo_a, 'do-a-thing', [{'a': 'a_1'}],
             user=config['role_user']['token'],
             reviewer=config['role_reviewer']['token'],
         )
     with repo_b:
-        make_branch(repo_b, 'master', 'initial', {'a': 'b_0'})
+        repo_b.make_commits(None, Commit('initial', tree={'a': 'b_0'}), ref='heads/master')
         prx_b = make_pr(repo_b, 'do-a-thing', [{'a': 'b_1'}],
             user=config['role_user']['token'],
             reviewer=config['role_reviewer']['token'],
@@ -166,7 +160,7 @@ def test_stage_match(env, project, repo_a, repo_b, config, page):
     assert get_related_pr_labels(pr_page(page, prx_a)) == [pr_b.display_name]
     assert get_related_pr_labels(pr_page(page, prx_b)) == [pr_a.display_name]
     with repo_a:
-        repo_a.post_status('staging.master', 'failure', 'legal/cla')
+        repo_a.post_status('staging.master', 'failure')
     env.run_crons()
 
     assert pr_a.state == 'error'
@@ -180,8 +174,7 @@ def test_stage_match(env, project, repo_a, repo_b, config, page):
     assert pr_a.staging_id and pr_b.staging_id
     for repo in [repo_a, repo_b]:
         with repo:
-            repo.post_status('staging.master', 'success', 'legal/cla')
-            repo.post_status('staging.master', 'success', 'ci/runbot')
+            repo.post_status('staging.master', 'success')
     env.run_crons()
     assert pr_a.state == 'merged'
     assert pr_b.state == 'merged'
@@ -201,8 +194,8 @@ def test_different_targets(env, project, repo_a, repo_b, config):
         'branch_ids': [(0, 0, {'name': 'other'})]
     })
     with repo_a:
-        make_branch(repo_a, 'master', 'initial', {'master': 'a_0'})
-        make_branch(repo_a, 'other', 'initial', {'other': 'a_0'})
+        repo_a.make_commits(None, Commit('initial', tree={'master': 'a_0'}), ref='heads/master')
+        repo_a.make_commits(None, Commit('initial', tree={'other': 'a_0'}), ref='heads/other')
         pr_a = make_pr(
             repo_a, 'do-a-thing', [{'mater': 'a_1'}],
             target='master',
@@ -210,8 +203,8 @@ def test_different_targets(env, project, repo_a, repo_b, config):
             reviewer=config['role_reviewer']['token'],
         )
     with repo_b:
-        make_branch(repo_b, 'master', 'initial', {'master': 'b_0'})
-        make_branch(repo_b, 'other', 'initial', {'other': 'b_0'})
+        repo_b.make_commits(None, Commit('initial', tree={'master': 'b_0'}), ref='heads/master')
+        repo_b.make_commits(None, Commit('initial', tree={'other': 'b_0'}), ref='heads/other')
         pr_b = make_pr(
             repo_b, 'do-a-thing', [{'other': 'b_1'}],
             target='other',
@@ -234,8 +227,7 @@ def test_different_targets(env, project, repo_a, repo_b, config):
 
     for r in [repo_a, repo_b]:
         with r:
-            r.post_status('staging.master', 'success', 'legal/cla')
-            r.post_status('staging.master', 'success', 'ci/runbot')
+            r.post_status('staging.master', 'success')
     env.run_crons()
     assert pr_a.state == 'merged'
 
@@ -249,7 +241,7 @@ def test_stage_different_statuses(env, project, repo_a, repo_b, config):
     })
 
     with repo_a:
-        make_branch(repo_a, 'master', 'initial', {'a': 'a_0'})
+        repo_a.make_commits(None, Commit('initial', tree={'a': 'a_0'}), ref='heads/master')
         pr_a = make_pr(
             repo_a, 'do-a-thing', [{'a': 'a_1'}],
             user=config['role_user']['token'],
@@ -257,17 +249,16 @@ def test_stage_different_statuses(env, project, repo_a, repo_b, config):
         )
         repo_a.post_status(pr_a.head, 'success', 'foo/bar')
     with repo_b:
-        make_branch(repo_b, 'master', 'initial', {'a': 'b_0'})
+        repo_b.make_commits(None, Commit('initial', tree={'a': 'b_0'}), ref='heads/master')
         [c] = repo_b.make_commits(
             'heads/master',
-            repo_b.Commit('some_commit\n\nSee also %s#%d' % (repo_a.name, pr_a.number), tree={'a': 'b_1'}),
+            repo_b.Commit(f'some_commit\n\nSee also {repo_a.name}#{pr_a.number:d}', tree={'a': 'b_1'}),
             ref='heads/do-a-thing'
         )
         pr_b = repo_b.make_pr(
             title="title", body="body", target='master', head='do-a-thing',
             token=config['role_user']['token'])
-        repo_b.post_status(c, 'success', 'ci/runbot')
-        repo_b.post_status(c, 'success', 'legal/cla')
+        repo_b.post_status(c, 'success')
         pr_b.post_comment('hansen r+', config['role_reviewer']['token'])
     env.run_crons()
     # since the labels are the same but the statuses on pr_b are not the
@@ -291,8 +282,7 @@ def test_stage_different_statuses(env, project, repo_a, repo_b, config):
     # do the actual merge to check for the Related header
     for repo in [repo_a, repo_b]:
         with repo:
-            repo.post_status('staging.master', 'success', 'legal/cla')
-            repo.post_status('staging.master', 'success', 'ci/runbot')
+            repo.post_status('staging.master', 'success')
             repo.post_status('staging.master', 'success', 'foo/bar')
     env.run_crons()
 
@@ -321,14 +311,14 @@ def test_unmatch_patch(env, project, repo_a, repo_b, config):
     """
     project.batch_limit = 1
     with repo_a:
-        make_branch(repo_a, 'master', 'initial', {'a': 'a_0'})
+        repo_a.make_commits(None, Commit('initial', tree={'a': 'a_0'}), ref='heads/master')
         pr_a = make_pr(
             repo_a, 'patch-1', [{'a': 'a_1'}],
             user=config['role_user']['token'],
             reviewer=config['role_reviewer']['token'],
         )
     with repo_b:
-        make_branch(repo_b, 'master', 'initial', {'a': 'b_0'})
+        repo_b.make_commits(None, Commit('initial', tree={'a': 'b_0'}), ref=f'heads/master')
         pr_b = make_pr(
             repo_b, 'patch-1', [{'a': 'b_1'}],
             user=config['role_user']['token'],
@@ -348,16 +338,16 @@ def test_sub_match(env, project, repo_a, repo_b, repo_c, config):
     """
     project.batch_limit = 1
     with repo_a: # no pr here
-        make_branch(repo_a, 'master', 'initial', {'a': 'a_0'})
+        repo_a.make_commits(None, Commit('initial', tree={'a': 'a_0'}), ref='heads/master')
     with repo_b:
-        make_branch(repo_b, 'master', 'initial', {'a': 'b_0'})
+        repo_b.make_commits(None, Commit('initial', tree={'a': 'b_0'}), ref='heads/master')
         pr_b = make_pr(
             repo_b, 'do-a-thing', [{'a': 'b_1'}],
             user=config['role_user']['token'],
             reviewer=config['role_reviewer']['token'],
         )
     with repo_c:
-        make_branch(repo_c, 'master', 'initial', {'a': 'c_0'})
+        repo_c.make_commits(None, Commit('initial', tree={'a': 'c_0'}), ref='heads/master')
         pr_c = make_pr(
             repo_c, 'do-a-thing', [{'a': 'c_1'}],
             user=config['role_user']['token'],
@@ -410,8 +400,8 @@ def test_merge_fail(env, project, repo_a, repo_b, users, config):
     project.batch_limit = 1
 
     with repo_a, repo_b:
-        make_branch(repo_a, 'master', 'initial', {'a': 'a_0'})
-        make_branch(repo_b, 'master', 'initial', {'a': 'b_0'})
+        repo_a.make_commits(None, Commit('initial', tree={'a': 'a_0'}), ref='heads/master')
+        repo_b.make_commits(None, Commit('initial', tree={'a': 'b_0'}), ref='heads/master')
 
         # first set of matched PRs
         pr1a = make_pr(
@@ -476,14 +466,14 @@ def test_ff_fail(env, project, repo_a, repo_b, config):
     project.batch_limit = 1
 
     with repo_a, repo_b:
-        root_a = make_branch(repo_a, 'master', 'initial', {'a': 'a_0'})
+        [root_a] = repo_a.make_commits(None, Commit('initial', tree={'a': 'a_0'}), ref='heads/master')
         make_pr(
             repo_a, 'do-a-thing', [{'a': 'a_1'}],
             user=config['role_user']['token'],
             reviewer=config['role_reviewer']['token'],
         )
 
-        make_branch(repo_b, 'master', 'initial', {'a': 'b_0'})
+        repo_b.make_commits(None, Commit('initial', tree={'a': 'b_0'}), ref=f'heads/master')
         make_pr(
             repo_b, 'do-a-thing', [{'a': 'b_1'}],
             user=config['role_user']['token'],
@@ -497,10 +487,8 @@ def test_ff_fail(env, project, repo_a, repo_b, config):
     assert repo_b.commit('heads/master').id == cn
 
     with repo_a, repo_b:
-        repo_a.post_status('heads/staging.master', 'success', 'ci/runbot')
-        repo_a.post_status('heads/staging.master', 'success', 'legal/cla')
-        repo_b.post_status('heads/staging.master', 'success', 'ci/runbot')
-        repo_b.post_status('heads/staging.master', 'success', 'legal/cla')
+        repo_a.post_status('heads/staging.master', 'success')
+        repo_b.post_status('heads/staging.master', 'success')
     env.run_crons('runbot_merge.merge_cron', 'runbot_merge.staging_cron')
     assert repo_b.commit('heads/master').id == cn,\
         "B should still be at the conflicting commit"
@@ -519,7 +507,7 @@ class TestCompanionsNotReady:
         """
         project.batch_limit = 1
         with repo_a, repo_b:
-            make_branch(repo_a, 'master', 'initial', {'a': 'a_0'})
+            repo_a.make_commits(None, Commit('initial', tree={'a': 'a_0'}), ref='heads/master')
             # pr_a is born ready
             p_a = make_pr(
                 repo_a, 'do-a-thing', [{'a': 'a_1'}],
@@ -527,7 +515,7 @@ class TestCompanionsNotReady:
                 reviewer=config['role_reviewer']['token'],
             )
 
-            make_branch(repo_b, 'master', 'initial', {'a': 'b_0'})
+            repo_b.make_commits(None, Commit('initial', tree={'a': 'b_0'}), ref='heads/master')
             p_b = make_pr(
                 repo_b, 'do-a-thing', [{'a': 'b_1'}],
                 user=config['role_user']['token'],
@@ -574,21 +562,21 @@ class TestCompanionsNotReady:
         """
         project.batch_limit = 1
         with repo_a, repo_b, repo_c:
-            make_branch(repo_a, 'master', 'initial', {'f': 'a0'})
+            repo_a.make_commits(None, Commit('initial', tree={'f': 'a0'}), ref='heads/master')
             pr_a = make_pr(
                 repo_a, 'a-thing', [{'f': 'a1'}],
                 user=config['role_user']['token'],
                 reviewer=None,
             )
 
-            make_branch(repo_b, 'master', 'initial', {'f': 'b0'})
+            repo_b.make_commits(None, Commit('initial', tree={'f': 'b0'}), ref='heads/master')
             pr_b = make_pr(
                 repo_b, 'a-thing', [{'f': 'b1'}],
                 user=config['role_user']['token'],
                 reviewer=config['role_reviewer']['token'],
             )
 
-            make_branch(repo_c, 'master', 'initial', {'f': 'c0'})
+            repo_c.make_commits(None, Commit('initial', tree={'f': 'c0'}), ref='heads/master')
             pr_c = make_pr(
                 repo_c, 'a-thing', [{'f': 'c1'}],
                 user=config['role_user']['token'],
@@ -614,21 +602,21 @@ class TestCompanionsNotReady:
         """
         project.batch_limit = 1
         with repo_a, repo_b, repo_c:
-            make_branch(repo_a, 'master', 'initial', {'f': 'a0'})
+            repo_a.make_commits(None, Commit('initial', tree={'f': 'a0'}), ref='heads/master')
             pr_a = make_pr(
                 repo_a, 'a-thing', [{'f': 'a1'}],
                 user=config['role_user']['token'],
                 reviewer=None,
             )
 
-            make_branch(repo_b, 'master', 'initial', {'f': 'b0'})
+            repo_b.make_commits(None, Commit('initial', tree={'f': 'b0'}), ref='heads/master')
             pr_b = make_pr(
                 repo_b, 'a-thing', [{'f': 'b1'}],
                 user=config['role_user']['token'],
                 reviewer=config['role_reviewer']['token'],
             )
 
-            make_branch(repo_c, 'master', 'initial', {'f': 'c0'})
+            repo_c.make_commits(None, Commit('initial', tree={'f': 'c0'}), ref='heads/master')
             pr_c = make_pr(
                 repo_c, 'a-thing', [{'f': 'c1'}],
                 user=config['role_user']['token'],
@@ -640,19 +628,13 @@ class TestCompanionsNotReady:
         assert pr_b.comments == [
             (users['reviewer'], 'hansen r+'),
             seen(env, pr_b, users),
-            (users['user'], "@%s @%s linked pull request(s) %s#%d not ready. Linked PRs are not staged until all of them are ready." % (
-                users['user'], users['reviewer'],
-                repo_a.name, pr_a.number
-            ))
+            (users['user'], f"@{users['user']} @{users['reviewer']} linked pull request(s) {repo_a.name}#{pr_a.number} not ready. Linked PRs are not staged until all of them are ready.")
         ]
         assert pr_c.comments == [
             (users['reviewer'], 'hansen r+'),
             seen(env, pr_c, users),
             (users['user'],
-             "@%s @%s linked pull request(s) %s#%d not ready. Linked PRs are not staged until all of them are ready." % (
-                 users['user'], users['reviewer'],
-                 repo_a.name, pr_a.number
-             ))
+             f"@{users['user']} @{users['reviewer']} linked pull request(s) {repo_a.name}#{pr_a.number} not ready. Linked PRs are not staged until all of them are ready.")
         ]
 
 def test_other_failed(env, project, repo_a, repo_b, users, config):
@@ -661,7 +643,7 @@ def test_other_failed(env, project, repo_a, repo_b, users, config):
     message
     """
     with repo_a, repo_b:
-        make_branch(repo_a, 'master', 'initial', {'a': 'a_0'})
+        repo_a.make_commits(None, Commit('initial', tree={'a': 'a_0'}), ref='heads/master')
         # pr_a is born ready
         pr_a = make_pr(
             repo_a, 'do-a-thing', [{'a': 'a_1'}],
@@ -669,17 +651,15 @@ def test_other_failed(env, project, repo_a, repo_b, users, config):
             reviewer=config['role_reviewer']['token'],
         )
 
-        make_branch(repo_b, 'master', 'initial', {'a': 'b_0'})
+        repo_b.make_commits(None, Commit('initial', tree={'a': 'b_0'}), ref='heads/master')
     env.run_crons()
 
     pr = to_pr(env, pr_a)
     assert pr.staging_id
 
     with repo_a, repo_b:
-        repo_a.post_status('heads/staging.master', 'success', 'legal/cla')
-        repo_a.post_status('heads/staging.master', 'success', 'ci/runbot', target_url="http://example.org/a")
-        repo_b.post_status('heads/staging.master', 'success', 'legal/cla')
-        repo_b.post_status('heads/staging.master', 'failure', 'ci/runbot', target_url="http://example.org/b")
+        repo_a.post_status('heads/staging.master', 'success', target_url="http://example.org/a")
+        repo_b.post_status('heads/staging.master', 'failure', target_url="http://example.org/b")
     env.run_crons()
 
     sth = repo_b.commit('heads/staging.master').id
@@ -688,7 +668,7 @@ def test_other_failed(env, project, repo_a, repo_b, users, config):
     assert pr_a.comments == [
         (users['reviewer'], 'hansen r+'),
         seen(env, pr_a, users),
-        (users['user'], '@%s @%s staging failed: ci/runbot on %s (view more at http://example.org/b)' % (
+        (users['user'], '@%s @%s staging failed: default on %s (view more at http://example.org/b)' % (
             users['user'], users['reviewer'],
             sth
         ))
@@ -702,8 +682,8 @@ class TestMultiBatches:
         project.batch_limit = 3
 
         with repo_a, repo_b:
-            make_branch(repo_a, 'master', 'initial', {'a': 'a0'})
-            make_branch(repo_b, 'master', 'initial', {'b': 'b0'})
+            repo_a.make_commits(None, Commit('initial', tree={'a': 'a0'}), ref='heads/master')
+            repo_b.make_commits(None, Commit('initial', tree={'b': 'b0'}), ref='heads/master')
 
             prs = [(
                 a and make_pr(repo_a, 'batch{}'.format(i), [{'a{}'.format(i): 'a{}'.format(i)}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token']),
@@ -735,8 +715,8 @@ class TestMultiBatches:
         """ If a staging fails, it should get split properly across repos
         """
         with repo_a, repo_b:
-            make_branch(repo_a, 'master', 'initial', {'a': 'a0'})
-            make_branch(repo_b, 'master', 'initial', {'b': 'b0'})
+            repo_a.make_commits(None, Commit('initial', tree={'a': 'a0'}), ref='heads/master')
+            repo_b.make_commits(None, Commit('initial', tree={'b': 'b0'}), ref='heads/master')
 
             prs = [(
                 a and make_pr(repo_a, 'batch{}'.format(i), [{'a{}'.format(i): 'a{}'.format(i)}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token']),
@@ -757,8 +737,7 @@ class TestMultiBatches:
         # mark b.staging as failed -> should create two splits with (0, 1)
         # and (2, 3, 4) and stage the first one
         with repo_b:
-            repo_b.post_status('heads/staging.master', 'success', 'legal/cla')
-            repo_b.post_status('heads/staging.master', 'failure', 'ci/runbot')
+            repo_b.post_status('heads/staging.master', 'failure')
         env.run_crons()
 
         assert not st0.active
@@ -782,8 +761,8 @@ def test_urgent(env, repo_a, repo_b, config):
     pair being prioritized
     """
     with repo_a, repo_b:
-        make_branch(repo_a, 'master', 'initial', {'a0': 'a'})
-        make_branch(repo_b, 'master', 'initial', {'b0': 'b'})
+        repo_a.make_commits(None, Commit('initial', tree={'a0': 'a'}), ref='heads/master')
+        repo_b.make_commits(None, Commit('initial', tree={'b0': 'b'}), ref='heads/master')
 
         pr_a = make_pr(repo_a, 'batch', [{'a1': 'a'}, {'a2': 'a'}], user=config['role_user']['token'], reviewer=None, statuses=[])
         pr_b = make_pr(repo_b, 'batch', [{'b1': 'b'}, {'b2': 'b'}], user=config['role_user']['token'], reviewer=None, statuses=[])
@@ -815,7 +794,7 @@ def test_urgent(env, repo_a, repo_b, config):
 class TestBlocked:
     def test_merge_method(self, env, repo_a, config):
         with repo_a:
-            make_branch(repo_a, 'master', 'initial', {'a0': 'a'})
+            repo_a.make_commits(None, Commit('initial', tree={'a0': 'a'}), ref='heads/master')
 
             pr = make_pr(repo_a, 'A', [{'a1': 'a'}, {'a2': 'a'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'],)
         env.run_crons()
@@ -829,8 +808,8 @@ class TestBlocked:
 
     def test_linked_closed(self, env, repo_a, repo_b, config):
         with repo_a, repo_b:
-            make_branch(repo_a, 'master', 'initial', {'a0': 'a'})
-            make_branch(repo_b, 'master', 'initial', {'b0': 'b'})
+            repo_a.make_commits(None, Commit('initial', tree={'a0': 'a'}), ref='heads/master')
+            repo_b.make_commits(None, Commit('initial', tree={'b0': 'b'}), ref='heads/master')
 
             pr = make_pr(repo_a, 'xxx', [{'a1': 'a'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'],)
             b = make_pr(repo_b, 'xxx', [{'b1': 'b'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'], statuses=[])
@@ -843,17 +822,15 @@ class TestBlocked:
 
     def test_linked_merged(self, env, repo_a, repo_b, config):
         with repo_a, repo_b:
-            make_branch(repo_a, 'master', 'initial', {'a0': 'a'})
-            make_branch(repo_b, 'master', 'initial', {'b0': 'b'})
+            repo_a.make_commits(None, Commit('initial', tree={'a0': 'a'}), ref='heads/master')
+            repo_b.make_commits(None, Commit('initial', tree={'b0': 'b'}), ref='heads/master')
 
             b = make_pr(repo_b, 'xxx', [{'b1': 'b'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'],)
         env.run_crons() # stage b and c
 
         with repo_a, repo_b:
-            repo_a.post_status('heads/staging.master', 'success', 'legal/cla')
-            repo_a.post_status('heads/staging.master', 'success', 'ci/runbot')
-            repo_b.post_status('heads/staging.master', 'success', 'legal/cla')
-            repo_b.post_status('heads/staging.master', 'success', 'ci/runbot')
+            repo_a.post_status('heads/staging.master', 'success')
+            repo_b.post_status('heads/staging.master', 'success')
         env.run_crons() # merge b and c
         assert to_pr(env, b).state == 'merged'
 
@@ -871,8 +848,8 @@ class TestBlocked:
         * A is not blocked if B.skipci
         """
         with repo_a, repo_b:
-            make_branch(repo_a, 'master', 'initial', {'a0': 'a'})
-            make_branch(repo_b, 'master', 'initial', {'b0': 'b'})
+            repo_a.make_commits(None, Commit('initial', tree={'a0': 'a'}), ref='heads/master')
+            repo_b.make_commits(None, Commit('initial', tree={'b0': 'b'}), ref='heads/master')
 
             a = make_pr(repo_a, 'xxx', [{'a1': 'a'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'],)
             b = make_pr(repo_b, 'xxx', [{'b1': 'b'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'], statuses=[])
@@ -896,9 +873,9 @@ def test_different_branches(env, project, repo_a, repo_b, config):
     env['runbot_merge.repository'].search([('name', '=', repo_b.name)])\
         .branch_filter = '[("name", "=", "master")]'
     with repo_a, repo_b:
-        make_branch(repo_a, 'dev', 'initial', {'a': '0'})
-        make_branch(repo_a, 'master', 'initial', {'b': '0'})
-        make_branch(repo_b, 'master', 'initial', {'b': '0'})
+        repo_a.make_commits(None, Commit('initial', tree={'a': '0'}), ref='heads/dev')
+        repo_a.make_commits(None, Commit('initial', tree={'b': '0'}), ref='heads/master')
+        repo_b.make_commits(None, Commit('initial', tree={'b': '0'}), ref='heads/master')
 
         pr_a = make_pr(
             repo_a, 'xxx', [{'a': '1'}],
@@ -910,8 +887,7 @@ def test_different_branches(env, project, repo_a, repo_b, config):
 
     with repo_a:
         pr_a.post_comment('hansen r+', config['role_reviewer']['token'])
-        repo_a.post_status('heads/staging.dev', 'success', 'legal/cla')
-        repo_a.post_status('heads/staging.dev', 'success', 'ci/runbot')
+        repo_a.post_status('heads/staging.dev', 'success')
     env.run_crons()
 
     assert to_pr(env, pr_a).state == 'merged'
@@ -990,9 +966,9 @@ class TestSubstitutions:
         repo_b_id.substitutions = r"/.+:/%s:/" % repo_a.owner
 
         with repo_a:
-            make_branch(repo_a, 'master', 'initial', {'a': '0'})
+            repo_a.make_commits(None, Commit('initial', tree={'a': '0'}), ref='heads/master')
         with repo_b:
-            make_branch(repo_b, 'master', 'initial', {'b': '0'})
+            repo_b.make_commits(None, Commit('initial', tree={'b': '0'}), ref='heads/master')
 
         # policy is that repo_a PRs are created in the same repo while repo_b PRs
         # are created in personal forks
@@ -1007,24 +983,16 @@ class TestSubstitutions:
                 target='master', head='%s:abranch' % b_fork.owner
             )
 
-        pra_id = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo_a.name),
-            ('number', '=', pra.number)
-        ])
-        prb_id = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo_b.name),
-            ('number', '=', prb.number)
-        ])
+        pra_id = to_pr(env, pra)
+        prb_id = to_pr(env, prb)
         assert pra_id.label.endswith(':abranch')
         assert prb_id.label.endswith(':abranch')
 
         with repo_a, repo_b:
-            repo_a.post_status(pra.head, 'success', 'legal/cla')
-            repo_a.post_status(pra.head, 'success', 'ci/runbot')
+            repo_a.post_status(pra.head, 'success')
             pra.post_comment('hansen r+', config['role_reviewer']['token'])
 
-            repo_b.post_status(prb.head, 'success', 'legal/cla')
-            repo_b.post_status(prb.head, 'success', 'ci/runbot')
+            repo_b.post_status(prb.head, 'success')
             prb.post_comment('hansen r+', config['role_reviewer']['token'])
         env.run_crons()
 
@@ -1194,18 +1162,15 @@ def test_freeze_complete(env, project, repo_a, repo_b, repo_c, users, config):
 
     with repo_a:
         pr_required_a.post_comment('hansen r+', config['role_reviewer']['token'])
-        repo_a.post_status(pr_required_a.head, 'success', 'ci/runbot')
-        repo_a.post_status(pr_required_a.head, 'success', 'legal/cla')
+        repo_a.post_status(pr_required_a.head, 'success')
     with repo_c:
         pr_required_c.post_comment('hansen r+', config['role_reviewer']['token'])
-        repo_c.post_status(pr_required_c.head, 'success', 'ci/runbot')
-        repo_c.post_status(pr_required_c.head, 'success', 'legal/cla')
+        repo_c.post_status(pr_required_c.head, 'success')
     env.run_crons()
 
     for repo in [repo_a, repo_b, repo_c]:
         with repo:
-            repo.post_status('staging.master', 'success', 'ci/runbot')
-            repo.post_status('staging.master', 'success', 'legal/cla')
+            repo.post_status('staging.master', 'success')
     env.run_crons()
 
     assert to_pr(env, pr_required_a).state == 'merged'
@@ -1475,8 +1440,8 @@ def test_cancel_staging(env, project, repo_a, repo_b, users, config):
     should get cancelled if and when the batch transitions to unblocked
     """
     with repo_a, repo_b:
-        make_branch(repo_a, 'master', 'initial', {'a': '1'})
-        make_branch(repo_b, 'master', 'initial', {'b': '1'})
+        repo_a.make_commits(None, Commit('initial', tree={'a': '1'}), ref='heads/master')
+        repo_b.make_commits(None, Commit('initial', tree={'b': '1'}), ref='heads/master')
 
         pr_a = make_pr(repo_a, 'batch', [{'a': '2'}], user=config['role_user']['token'], statuses=[], reviewer=None)
         pr_b = make_pr(repo_b, 'batch', [{'b': '2'}], user=config['role_user']['token'], statuses=[], reviewer=None)
@@ -1504,8 +1469,7 @@ def test_cancel_staging(env, project, repo_a, repo_b, users, config):
     assert a_id.state == 'approved'
     assert lone_id.staging_id == st
     with repo_a:
-        repo_a.post_status(a_id.head, 'success', 'ci/runbot')
-        repo_a.post_status(a_id.head, 'success', 'legal/cla')
+        repo_a.post_status(a_id.head, 'success')
     env.run_crons()
     assert a_id.state == 'ready'
     assert lone_id.staging_id == st
@@ -1516,8 +1480,7 @@ def test_cancel_staging(env, project, repo_a, repo_b, users, config):
     assert b_id.state == 'approved'
     assert lone_id.staging_id == st
     with repo_b:
-        repo_b.post_status(b_id.head, 'success', 'ci/runbot')
-        repo_b.post_status(b_id.head, 'success', 'legal/cla')
+        repo_b.post_status(b_id.head, 'success')
     assert b_id.state == 'approved'
     assert lone_id.staging_id == st
     env.run_crons()

From fa2bba3cb9d5d353f2f6f38f451ea0b30cdeac04 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 12 Feb 2024 08:46:44 +0100
Subject: [PATCH 091/192] [CHG] runbot_merge: don't reset cancel_staging on r-

Also send skipchecks removal to the PR being r-'d, as sending it to a
random PR of the batch doesn't really make sense?
---
 runbot_merge/models/pull_requests.py |  4 +---
 runbot_merge/tests/test_basic.py     | 11 ++---------
 2 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 02eab8c4..4413857a 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -700,8 +700,6 @@ class PullRequests(models.Model):
                     else:
                         msg = self._approve(author, login)
                 case commands.Reject() if is_author:
-                    if self.batch_id.cancel_staging:
-                        self.batch_id.cancel_staging = False
                     if self.batch_id.skipchecks or self.reviewed_by:
                         if self.error:
                             self.error = False
@@ -712,7 +710,7 @@ class PullRequests(models.Model):
                             self.env.ref("runbot_merge.command.unapprove.p0")._send(
                                 repository=self.repository,
                                 pull_request=self.number,
-                                format_args={'user': login, 'pr': self.batch_id.prs[:1]},
+                                format_args={'user': login, 'pr': self},
                             )
                         self.unstage("unreviewed (r-) by %s", login)
                     else:
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index dc458b34..d7653513 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -2880,17 +2880,10 @@ class TestBatching(object):
         assert p_01.state == 'opened'
         assert p_01.priority == 'alone'
         assert p_01.skipchecks == False
-        assert p_01.cancel_staging == False
+        assert p_01.cancel_staging == True
 
-        p_01.batch_id.cancel_staging = True
-        # FIXME: cancel_staging should only cancel when the PR is or
-        #        transitions to ready
-        # assert staging_4.active
-        # re-staging, should not be necessary
-        env.run_crons()
+        assert staging_4.active, "staging should not be disabled"
 
-        staging_5 = ensure_one(sm_all.staging_id)
-        assert staging_5.active
         # cause the PR to become ready the normal way
         with repo:
             pr01.post_comment("hansen r+", config['role_reviewer']['token'])

From f97502e503cb52c0000ad9b153c4ddb7482c3a6d Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 24 May 2024 08:36:19 +0200
Subject: [PATCH 092/192] [IMP] runbot_merge: make skipchecks impact PR state

It's a bit weird and inconsistent to have a PR being staged while
unreviewed or unapproved or w/e. If we compute the state based on
skipchecks then everything is consistent.

Also remove the implicit override of all statuses when explicitly
marking the pr as `ready`, it risks creating difficult to understand
states, and it's unnecessary since `skipchecks` gets set.

Also as with setting skipchecks, sets the current user as reviewer on
all PRs of the batch without a reviewer.
---
 runbot_merge/models/batch.py         |  2 ++
 runbot_merge/models/pull_requests.py | 28 ++++++++++++----------------
 runbot_merge/tests/test_basic.py     |  4 ++--
 runbot_merge/tests/test_oddities.py  |  2 +-
 4 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index c7b82b30..cb378940 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -28,6 +28,8 @@ class Batch(models.Model):
     ], required=True, default="default", string="Forward Port Policy")
 
     merge_date = fields.Datetime(tracking=True)
+    # having skipchecks skip both validation *and approval* makes sense because
+    # it's batch-wise, having to approve individual PRs is annoying
     skipchecks = fields.Boolean(
         string="Skips Checks",
         default=False, tracking=True,
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 4413857a..f2adaa9d 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -929,8 +929,11 @@ class PullRequests(models.Model):
                     st = 'pending'
             pr.status = st
 
-    # closed, merged, error should be exclusive, so this should probably be a selection
-    @api.depends("status", "reviewed_by", 'batch_id.merge_date', "closed", "error")
+    @api.depends(
+        "status", "reviewed_by", "closed", "error" ,
+        "batch_id.merge_date",
+        "batch_id.skipchecks",
+    )
     def _compute_state(self):
         for pr in self:
             if pr.batch_id.merge_date:
@@ -939,6 +942,8 @@ class PullRequests(models.Model):
                 pr.state = "closed"
             elif pr.error:
                 pr.state = "error"
+            elif pr.batch_id.skipchecks: # skipchecks behaves as both approval and status override
+                pr.state = "ready"
             else:
                 states = ("opened", "approved", "validated", "ready")
                 pr.state = states[bool(pr.reviewed_by) | ((pr.status == "success") << 1)]
@@ -1079,22 +1084,13 @@ class PullRequests(models.Model):
 
         # when explicitly marking a PR as ready
         if vals.get('state') == 'ready':
-            # skip checks anyway
+            # skip validation
             self.batch_id.skipchecks = True
-            # if the state is forced to ready, set current user as reviewer
-            # and override all statuses
+            # mark current user as reviewer
             vals.setdefault('reviewed_by', self.env.user.partner_id.id)
-            # override all known statuses just for safety
-            vals.setdefault('overrides', json.dumps({
-                st.context: {
-                    'state': 'success',
-                    'target_url': None,
-                    'description': f"Forced by @{self.env.user.partner_id.github_login}",
-                }
-                for st in self.env['runbot_merge.repository.status'].search([
-                    ('prs', '=', True),
-                ])
-            }))
+            for p in self.batch_id.prs - self:
+                if not p.reviewed_by:
+                    p.reviewed_by = self.env.user.partner_id.id
 
         for pr in self:
             if (t := vals.get('target')) is not None and pr.target.id != t:
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index d7653513..77fb3859 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -2810,7 +2810,7 @@ class TestBatching(object):
             pr01 = self._pr(repo, 'Urgent1', [{'n': 'n'}, {'o': 'o'}], user=config['role_user']['token'], reviewer=None, statuses=[])
             pr01.post_comment('hansen NOW! rebase-merge', config['role_reviewer']['token'])
         p_01 = to_pr(env, pr01)
-        assert p_01.state == 'approved'
+        assert p_01.state == 'ready'
         assert p_01.priority == 'alone'
         assert p_01.skipchecks == True
 
@@ -2862,7 +2862,7 @@ class TestBatching(object):
             )
         env.run_crons()
         assert not staging_3.active
-        assert p_01.state == 'opened'
+        assert p_01.state == 'ready'
         assert p_01.priority == 'alone'
         assert p_01.skipchecks == True
         assert p_01.staging_id.active
diff --git a/runbot_merge/tests/test_oddities.py b/runbot_merge/tests/test_oddities.py
index 43448656..b5d9d5b2 100644
--- a/runbot_merge/tests/test_oddities.py
+++ b/runbot_merge/tests/test_oddities.py
@@ -273,6 +273,6 @@ def test_force_ready(env, make_repo, project, setreviewers, config):
     pr_id.state = 'ready'
 
     assert pr_id.state == 'ready'
+    assert pr_id.status == 'pending'
     reviewer = env['res.users'].browse([env._uid]).partner_id
     assert pr_id.reviewed_by == reviewer
-    assert pr_id.overrides

From dae046708f780d2bee80d7c53da72b8153334c91 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 12 Feb 2024 08:51:23 +0100
Subject: [PATCH 093/192] [IMP] runbot_merge: make batch blocked message more
 precise

In case of PRs not being ready, don't just say the PRs are waiting for
CI even though they might be unreviewed, and make a difference
between *waiting* for CI (pending) and having failed CI.
---
 runbot_merge/models/batch.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index cb378940..8f819349 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -90,9 +90,14 @@ class Batch(models.Model):
             elif not batch.skipchecks and (unready := batch.prs.filtered(
                 lambda p: not (p.reviewed_by and p.status == "success")
             )):
-                batch.blocked = "Pull request(s) %s waiting for CI." % ', '.join(
-                    p.display_name for p in unready
-                )
+                unreviewed = ', '.join(unready.filtered(lambda p: not p.reviewed_by).mapped('display_name'))
+                unvalidated = ', '.join(unready.filtered(lambda p: p.status == 'pending').mapped('display_name'))
+                failed = ', '.join(unready.filtered(lambda p: p.status == 'failure').mapped('display_name'))
+                batch.blocked = "Pull request(s) %s." % ', '.join(filter(None, [
+                    unreviewed and f"{unreviewed} are waiting for review",
+                    unvalidated and f"{unvalidated} are waiting for CI",
+                    failed and f"{failed} have failed CI",
+                ]))
             else:
                 if batch.blocked and batch.cancel_staging:
                     batch.target.active_staging_id.cancel(

From a4a067e7e9e9475893297528652081a49c97923f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 21 May 2024 15:45:53 +0200
Subject: [PATCH 094/192] [CHG] *: move forward-porting over to batches

Thank god I have a bunch of tests because once again I forgot / missed
a bunch of edge cases in doing the conversion, which the tests
caught (sadly that means I almost certainly broke a few untested edge
cases).

Important notes:

Handling of parent links
------------------------

Unlike PRs, batches don't lose their parent info ever, the link is
permanent, which is convenient to trawl through a forward port
(currently implemented very inefficiently, maybe we'll optimise that
in the future).

However this means the batch having a parent and the batch's PRs
having parents are slightly different informations, one of the edge
cases I missed is that of conflicting PRs, which are deparented and
have to be merged by hand before being forward ported further, I had
originally replaced the checks on a pr and its sibling having parents
by just the batch.

Batches & targets
-----------------

Batches were originally concepted as being fixed to a target and PRs
having that target, a PR being retargeted would move it from one batch
to an other.

As it turns out this does not work in the case where people retarget
forward-port PRs, which I know they do because #551
(2337bd85186de000d074e5a7178cfeb110d15de7). I could not think of a
good way to handle this issue as is, so scrapped the moving PRs thing,
instead one of the coherence checks of a batch being ready is that all
its PRs have the same target, and a batch only has a target if all its
PRs have the same target.

It's possible for somewhat odd effects to arise, notably if a PR is
closed (removed from batch), the other PRs are retargeted, and the new
PR is reopened, it will now be on a separate batch even if it also
gets retargeted. This is weird. I don't quite know how I should handle
it, maybe batches could merge if they have the same target and label?
however batches don't currently have a label so...

Improve limits
--------------

Keep limits on the PRs rather than lift them on the batchL if we can
add/remove PRs of batches having different limits on different PRs of
the same batch is reasonable.

Also leave limit unset by default: previously, the limit was eagerly
set to the tip (accessible) branch. That doesn't really seem
necessary, so stop doing that.

Also remove completely unnecessary `max` when trying to find a PR's
next target: `root` is either `self` or `self.source_id`, so it should
not be possible for that to have a later target.

And for now ensure the limits are consistent per batch: a PR defaults
to the limit of their batch-mate if they don't have one, and if a
limit is set via command it's set on all PRs of a batch.

This commit does not allow differential limits via commands, they are
allowed via the backend but not really tested. The issue is mostly
that it's not clear what the UX should look like to have clear and not
super error prone interactions. So punt on it for now, and hopefully
there's no hole I missed which will create inconsistent batches.
---
 forwardport/models/forwardport.py             |  21 +-
 forwardport/models/project.py                 | 467 +-----------------
 forwardport/tests/test_limit.py               |   2 +-
 forwardport/tests/test_overrides.py           |   2 +-
 forwardport/tests/test_simple.py              |  12 +-
 ..._merge.pull_requests.feedback.template.csv |   2 +-
 runbot_merge/models/batch.py                  | 422 +++++++++++++++-
 runbot_merge/models/project.py                |  11 +-
 runbot_merge/models/pull_requests.py          | 201 +++++++-
 runbot_merge/views/mergebot.xml               |   2 +-
 10 files changed, 628 insertions(+), 514 deletions(-)

diff --git a/forwardport/models/forwardport.py b/forwardport/models/forwardport.py
index ee3d9b25..00dac319 100644
--- a/forwardport/models/forwardport.py
+++ b/forwardport/models/forwardport.py
@@ -82,25 +82,28 @@ class ForwardPortTasks(models.Model, Queue):
     def _process_item(self):
         batch = self.batch_id
         sentry_sdk.set_tag('forward-porting', batch.prs.mapped('display_name'))
-        newbatch = batch.prs._port_forward()
+        newbatch = batch._port_forward()
 
         if not newbatch:  # reached end of seq (or batch is empty)
             # FIXME: or configuration is fucky so doesn't want to FP (maybe should error and retry?)
             _logger.info(
-                "Processed %s (from %s): %s (%s) -> end of the sequence",
-                self.id, self.source,
-                batch, batch.prs
+                "Processed %s from %s (%s) -> end of the sequence",
+                batch, self.source, batch.prs.mapped('display_name'),
             )
             return
 
         _logger.info(
-            "Processed %s (from %s): %s (%s) -> %s (%s)",
-            self.id, self.source,
-            batch, batch.prs,
-            newbatch, newbatch.prs,
+            "Processed %s from %s (%s) -> %s (%s)",
+            batch, self.source, ', '.join(batch.prs.mapped('display_name')),
+            newbatch, ', '.join(newbatch.prs.mapped('display_name')),
         )
-        # insert new batch in ancestry sequence unless conflict (= no parent)
+        # insert new batch in ancestry sequence
         if self.source == 'insert':
+            self.env['runbot_merge.batch'].search([
+                ('parent_id', '=', batch.id),
+                ('id', '!=', newbatch.id),
+            ]).parent_id = newbatch.id
+            # insert new PRs in ancestry sequence unless conflict (= no parent)
             for pr in newbatch.prs:
                 if not pr.parent_id:
                     break
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index d4a30f21..137e69ea 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -11,31 +11,22 @@ means PR creation is trickier (as mergebot assumes opened event will always
 lead to PR creation but fpbot wants to attach meaning to the PR when setting
 it up), ...
 """
-import ast
-import base64
-import contextlib
 import datetime
 import itertools
 import json
 import logging
 import operator
-import os
-import re
 import subprocess
 import tempfile
 import typing
-from operator import itemgetter
 from pathlib import Path
 
 import dateutil.relativedelta
-import psycopg2.errors
 import requests
 
 from odoo import models, fields, api
-from odoo.osv import expression
 from odoo.exceptions import UserError
-from odoo.tools.misc import topological_sort, groupby, Reverse
-from odoo.tools.sql import reverse_order
+from odoo.tools.misc import topological_sort, groupby
 from odoo.tools.appdirs import user_cache_dir
 from odoo.addons.base.models.res_partner import Partner
 from odoo.addons.runbot_merge import git
@@ -43,8 +34,6 @@ from odoo.addons.runbot_merge.models.pull_requests import Branch
 from odoo.addons.runbot_merge.models.stagings_create import Message
 
 
-footer = '\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n'
-
 DEFAULT_DELTA = dateutil.relativedelta.relativedelta(days=3)
 
 _logger = logging.getLogger('odoo.addons.forwardport')
@@ -72,7 +61,7 @@ class Project(models.Model):
         because no CI or CI failed), create followup, as if the branch had been
         originally disabled (and thus skipped over)
         """
-        PRs = self.env['runbot_merge.pull_requests']
+        Batch = self.env['runbot_merge.batch']
         for p in self:
             actives = previously_active_branches[p]
             for deactivated in p.branch_ids.filtered(lambda b: not b.active) & actives:
@@ -80,26 +69,27 @@ class Project(models.Model):
                 # and it doesn't have a child (e.g. CI failed), enqueue a forward
                 # port as if the now deactivated branch had been skipped over (which
                 # is the normal fw behaviour)
-                extant = PRs.search([
+                extant = Batch.search([
                     ('target', '=', deactivated.id),
-                    ('source_id.limit_id', '!=', deactivated.id),
-                    ('state', 'not in', ('closed', 'merged')),
+                    # FIXME: this should probably be *all* the PRs of the batch
+                    ('prs.limit_id', '!=', deactivated.id),
+                    ('merge_date', '=', False),
                 ])
-                for p in extant.with_context(force_fw=True):
-                    next_target = p.source_id._find_next_target(p)
+                for b in extant.with_context(force_fw=True):
+                    next_target = b._find_next_target()
                     # should not happen since we already filtered out limits
                     if not next_target:
                         continue
 
                     # check if it has a descendant in the next branch, if so skip
-                    if PRs.search_count([
-                        ('source_id', '=', p.source_id.id),
+                    if Batch.search_count([
+                        ('parent_id', '=', b.id),
                         ('target', '=', next_target.id)
                     ]):
                         continue
 
                     # otherwise enqueue a followup
-                    p._schedule_fp_followup()
+                    b._schedule_fp_followup()
 
     def _insert_intermediate_prs(self, branches_before):
         """If new branches have been added to the sequence inbetween existing
@@ -161,13 +151,6 @@ class Project(models.Model):
                     'source': 'insert',
                 })
 
-    def _forward_port_ordered(self, domain=()):
-        Branches = self.env['runbot_merge.branch']
-        return Branches.search(expression.AND([
-            [('project_id', '=', self.id)],
-            domain or [],
-        ]), order=reverse_order(Branches._order))
-
 class Repository(models.Model):
     _inherit = 'runbot_merge.repository'
 
@@ -201,12 +184,6 @@ class PullRequests(models.Model):
         if existing:
             return existing
 
-        if 'limit_id' not in vals:
-            branch = self.env['runbot_merge.branch'].browse(vals['target'])
-            repo = self.env['runbot_merge.repository'].browse(vals['repository'])
-            vals['limit_id'] = branch.project_id._forward_port_ordered(
-                ast.literal_eval(repo.branch_filter or '[]')
-            )[-1].id
         if vals.get('parent_id') and 'source_id' not in vals:
             vals['source_id'] = self.browse(vals['parent_id']).root_id.id
         return super().create(vals)
@@ -273,96 +250,6 @@ class PullRequests(models.Model):
             })
         return r
 
-    def _maybe_update_limit(self, limit: str) -> typing.Tuple[bool, str]:
-        limit_id = self.env['runbot_merge.branch'].with_context(active_test=False).search([
-            ('project_id', '=', self.repository.project_id.id),
-            ('name', '=', limit),
-        ])
-        if not limit_id:
-            return True, f"there is no branch {limit!r}, it can't be used as a forward port target."
-
-        if limit_id != self.target and not limit_id.active:
-            return True, f"branch {limit_id.name!r} is disabled, it can't be used as a forward port target."
-
-        # not forward ported yet, just acknowledge the request
-        if not self.source_id and self.state != 'merged':
-            self.limit_id = limit_id
-            if branch_key(limit_id) <= branch_key(self.target):
-                return False, "Forward-port disabled."
-            else:
-                return False, f"Forward-porting to {limit_id.name!r}."
-
-        # if the PR has been forwardported
-        prs = (self | self.forwardport_ids | self.source_id | self.source_id.forwardport_ids)
-        tip = max(prs, key=pr_key)
-        # if the fp tip was closed it's fine
-        if tip.state == 'closed':
-            return True, f"{tip.display_name} is closed, no forward porting is going on"
-
-        prs.limit_id = limit_id
-
-        real_limit = max(limit_id, tip.target, key=branch_key)
-
-        addendum = ''
-        # check if tip was queued for forward porting, try to cancel if we're
-        # supposed to stop here
-        if real_limit == tip.target and (task := self.env['forwardport.batches'].search([('batch_id', '=', tip.batch_id.id)])):
-            try:
-                with self.env.cr.savepoint():
-                    self.env.cr.execute(
-                        "SELECT FROM forwardport_batches "
-                        "WHERE id = %s FOR UPDATE NOWAIT",
-                        [task.id])
-            except psycopg2.errors.LockNotAvailable:
-                # row locked = port occurring and probably going to succeed,
-                # so next(real_limit) likely a done deal already
-                return True, (
-                    f"Forward port of {tip.display_name} likely already "
-                    f"ongoing, unable to cancel, close next forward port "
-                    f"when it completes.")
-            else:
-                self.env.cr.execute("DELETE FROM forwardport_batches WHERE id = %s", [task.id])
-
-        if real_limit != tip.target:
-            # forward porting was previously stopped at tip, and we want it to
-            # resume
-            if tip.state == 'merged':
-                self.env['forwardport.batches'].create({
-                    'batch_id': tip.batch_id.id,
-                    'source': 'fp' if tip.parent_id else 'merge',
-                })
-                resumed = tip
-            else:
-                resumed = tip._schedule_fp_followup()
-            if resumed:
-                addendum += f', resuming forward-port stopped at {tip.display_name}'
-
-        if real_limit != limit_id:
-            addendum += f' (instead of the requested {limit_id.name!r} because {tip.display_name} already exists)'
-
-        # get a "stable" root rather than self's to avoid divertences between
-        # PRs across a root divide (where one post-root would point to the root,
-        # and one pre-root would point to the source, or a previous root)
-        root = tip.root_id
-        # reference the root being forward ported unless we are the root
-        root_ref = '' if root == self else f' {root.display_name}'
-        msg = f"Forward-porting{root_ref} to {real_limit.name!r}{addendum}."
-        # send a message to the source & root except for self, if they exist
-        root_msg = f'Forward-porting to {real_limit.name!r} (from {self.display_name}).'
-        self.env['runbot_merge.pull_requests.feedback'].create([
-            {
-                'repository': p.repository.id,
-                'pull_request': p.number,
-                'message': root_msg,
-                'token_field': 'fp_github_token',
-            }
-            # send messages to source and root unless root is self (as it
-            # already gets the normal message)
-            for p in (self.source_id | root) - self
-        ])
-
-        return False, msg
-
     def _notify_ci_failed(self, ci):
         # only care about FP PRs which are not staged / merged yet
         # NB: probably ignore approved PRs as normal message will handle them?
@@ -378,89 +265,9 @@ class PullRequests(models.Model):
 
     def _validate(self, statuses):
         failed = super()._validate(statuses)
-        self._schedule_fp_followup()
+        self.batch_id._schedule_fp_followup()
         return failed
 
-    def _schedule_fp_followup(self):
-        _logger = logging.getLogger(__name__).getChild('forwardport.next')
-        # if the PR has a parent and is CI-validated, enqueue the next PR
-        scheduled = self.browse(())
-        for pr in self:
-            _logger.info('Checking if forward-port %s (%s)', pr.display_name, pr)
-            if not pr.parent_id:
-                _logger.info('-> no parent %s (%s)', pr.display_name, pr.parent_id)
-                continue
-            if not self.env.context.get('force_fw') and self.source_id.batch_id.fw_policy != 'skipci' and pr.state not in ['validated', 'ready']:
-                _logger.info('-> wrong state %s (%s)', pr.display_name, pr.state)
-                continue
-
-            # check if we've already forward-ported this branch
-            source = pr.source_id or pr
-            next_target = source._find_next_target(pr)
-            if not next_target:
-                _logger.info("-> forward port done (no next target)")
-                continue
-
-            if n := self.search([
-                ('source_id', '=', source.id),
-                ('target', '=', next_target.id),
-            ], limit=1):
-                _logger.info('-> already forward-ported (%s)', n.display_name)
-                continue
-
-            batch = pr.batch_id
-            # otherwise check if we already have a pending forward port
-            _logger.info("%s %s %s", pr.display_name, batch, ', '.join(batch.mapped('prs.display_name')))
-            if self.env['forwardport.batches'].search_count([('batch_id', '=', batch.id)]):
-                _logger.warning('-> already recorded')
-                continue
-
-            # check if batch-mate are all valid
-            mates = batch.prs
-            # wait until all of them are validated or ready
-            if not self.env.context.get('force_fw') and any(pr.source_id.batch_id.fw_policy != 'skipci' and pr.state not in ('validated', 'ready') for pr in mates):
-                _logger.info("-> not ready (%s)", [(pr.display_name, pr.state) for pr in mates])
-                continue
-
-            # check that there's no weird-ass state
-            if not all(pr.parent_id for pr in mates):
-                _logger.warning("Found a batch (%s) with only some PRs having parents, ignoring", mates)
-                continue
-            if self.search_count([('parent_id', 'in', mates.ids)]):
-                _logger.warning("Found a batch (%s) with only some of the PRs having children", mates)
-                continue
-
-            _logger.info('-> ok')
-            self.env['forwardport.batches'].create({
-                'batch_id': batch.id,
-                'source': 'fp',
-            })
-            scheduled |= pr
-        return scheduled
-
-    def _find_next_target(self, reference):
-        """ Finds the branch between target and limit_id which follows
-        reference
-        """
-        if reference.target == self.limit_id:
-            return
-        # NOTE: assumes even disabled branches are properly sequenced, would
-        #       probably be a good idea to have the FP view show all branches
-        branches = list(self.target.project_id
-            .with_context(active_test=False)
-            ._forward_port_ordered(ast.literal_eval(self.repository.branch_filter or '[]')))
-
-        # get all branches between max(root.target, ref.target) (excluded) and limit (included)
-        from_ = max(branches.index(self.target), branches.index(reference.target))
-        to_ = branches.index(self.limit_id)
-
-        # return the first active branch in the set
-        return next((
-            branch
-            for branch in branches[from_+1:to_+1]
-            if branch.active
-        ), None)
-
     def _commits_lazy(self):
         s = requests.Session()
         s.headers['Authorization'] = 'token %s' % self.repository.project_id.fp_github_token
@@ -490,216 +297,6 @@ class PullRequests(models.Model):
         return sorted(commits, key=lambda c: idx[c['sha']])
 
 
-    def _port_forward(self):
-        if not self:
-            return
-
-        all_sources = [(p.source_id or p) for p in self]
-        all_targets = [s._find_next_target(p) for s, p in zip(all_sources, self)]
-
-        ref = self[0]
-        base = all_sources[0]
-        target = all_targets[0]
-        if target is None:
-            _logger.info(
-                "Will not forward-port %s: no next target",
-                ref.display_name,
-            )
-            return  # QUESTION: do the prs need to be updated?
-
-        # check if the PRs have already been forward-ported: is there a PR
-        # with the same source targeting the next branch in the series
-        for source in all_sources:
-            if self.search_count([('source_id', '=', source.id), ('target', '=', target.id)]):
-                _logger.info("Will not forward-port %s: already ported", ref.display_name)
-                return
-
-        # check if all PRs in the batch have the same "next target" , bail if
-        # that's not the case as it doesn't make sense for forward one PR from
-        # a to b and a linked pr from a to c
-        different_target = next((t for t in all_targets if t != target), None)
-        if different_target:
-            different_pr = next(p for p, t in zip(self, all_targets) if t == different_target)
-            for pr, t in zip(self, all_targets):
-                linked, other = different_pr, different_target
-                if t != target:
-                    linked, other = ref, target
-                self.env.ref('runbot_merge.forwardport.failure.discrepancy')._send(
-                    repository=pr.repository,
-                    pull_request=pr.number,
-                    token_field='fp_github_token',
-                    format_args={'pr': pr, 'linked': linked, 'next': t.name, 'other': other.name},
-                )
-            _logger.warning(
-                "Cancelling forward-port of %s: found different next branches (%s)",
-                self, all_targets
-            )
-            return
-
-        proj = self.mapped('target.project_id')
-        if not proj.fp_github_token:
-            _logger.warning(
-                "Can not forward-port %s: no token on project %s",
-                ref.display_name,
-                proj.name
-            )
-            return
-
-        notarget = [p.repository.name for p in self if not p.repository.fp_remote_target]
-        if notarget:
-            _logger.error(
-                "Can not forward-port %s: repos %s don't have a remote configured",
-                self, ', '.join(notarget)
-            )
-            return
-
-        # take only the branch bit
-        new_branch = '%s-%s-%s-fw' % (
-            target.name,
-            base.refname,
-            # avoid collisions between fp branches (labels can be reused
-            # or conflict especially as we're chopping off the owner)
-            base64.urlsafe_b64encode(os.urandom(3)).decode()
-        )
-        # TODO: send outputs to logging?
-        conflicts = {}
-        with contextlib.ExitStack() as s:
-            for pr in self:
-                conflicts[pr], working_copy = pr._create_fp_branch(
-                    target, new_branch, s)
-
-                working_copy.push('target', new_branch)
-
-        gh = requests.Session()
-        gh.headers['Authorization'] = 'token %s' % proj.fp_github_token
-        has_conflicts = any(conflicts.values())
-        # problemo: this should forward port a batch at a time, if porting
-        # one of the PRs in the batch fails is huge problem, though this loop
-        # only concerns itself with the creation of the followup objects so...
-        new_batch = self.browse(())
-        self.env.cr.execute('LOCK runbot_merge_pull_requests IN SHARE MODE')
-        for pr in self:
-            owner, _ = pr.repository.fp_remote_target.split('/', 1)
-            source = pr.source_id or pr
-            root = pr.root_id
-
-            message = source.message + '\n\n' + '\n'.join(
-                "Forward-Port-Of: %s" % p.display_name
-                for p in root | source
-            )
-
-            title, body = re.match(r'(?P<title>[^\n]+)\n*(?P<body>.*)', message, flags=re.DOTALL).groups()
-            r = gh.post(f'https://api.github.com/repos/{pr.repository.name}/pulls', json={
-                'base': target.name,
-                'head': f'{owner}:{new_branch}',
-                'title': '[FW]' + (' ' if title[0] != '[' else '') + title,
-                'body': body
-            })
-            if not r.ok:
-                _logger.warning("Failed to create forward-port PR for %s, deleting branches", pr.display_name)
-                # delete all the branches this should automatically close the
-                # PRs if we've created any. Using the API here is probably
-                # simpler than going through the working copies
-                for repo in self.mapped('repository'):
-                    d = gh.delete(f'https://api.github.com/repos/{repo.fp_remote_target}/git/refs/heads/{new_branch}')
-                    if d.ok:
-                        _logger.info("Deleting %s:%s=success", repo.fp_remote_target, new_branch)
-                    else:
-                        _logger.warning("Deleting %s:%s=%s", repo.fp_remote_target, new_branch, d.text)
-                raise RuntimeError("Forwardport failure: %s (%s)" % (pr.display_name, r.text))
-
-            new_pr = self._from_gh(r.json())
-            _logger.info("Created forward-port PR %s", new_pr)
-            new_batch |= new_pr
-
-            # allows PR author to close or skipci
-            new_pr.write({
-                'merge_method': pr.merge_method,
-                'source_id': source.id,
-                # only link to previous PR of sequence if cherrypick passed
-                'parent_id': pr.id if not has_conflicts else False,
-                'detach_reason': "conflicts: {}".format(
-                    f'\n{conflicts[pr]}\n{conflicts[pr]}'.strip()
-                ) if has_conflicts else None,
-            })
-            if has_conflicts and pr.parent_id and pr.state not in ('merged', 'closed'):
-                self.env.ref('runbot_merge.forwardport.failure.conflict')._send(
-                    repository=pr.repository,
-                    pull_request=pr.number,
-                    token_field='fp_github_token',
-                    format_args={'source': source, 'pr': pr, 'new': new_pr, 'footer': footer},
-                )
-
-        for pr, new_pr in zip(self, new_batch):
-            (h, out, err, hh) = conflicts.get(pr) or (None, None, None, None)
-
-            if h:
-                sout = serr = ''
-                if out.strip():
-                    sout = f"\nstdout:\n```\n{out}\n```\n"
-                if err.strip():
-                    serr = f"\nstderr:\n```\n{err}\n```\n"
-
-                lines = ''
-                if len(hh) > 1:
-                    lines = '\n' + ''.join(
-                        '* %s%s\n' % (sha, ' <- on this commit' if sha == h else '')
-                        for sha in hh
-                    )
-                template = 'runbot_merge.forwardport.failure'
-                format_args = {
-                    'pr': new_pr,
-                    'commits': lines,
-                    'stdout': sout,
-                    'stderr': serr,
-                    'footer': footer,
-                }
-            elif has_conflicts:
-                template = 'runbot_merge.forwardport.linked'
-                format_args = {
-                    'pr': new_pr,
-                    'siblings': ', '.join(p.display_name for p in (new_batch - new_pr)),
-                    'footer': footer,
-                }
-            elif base._find_next_target(new_pr) is None:
-                ancestors = "".join(
-                    "* %s\n" % p.display_name
-                    for p in pr._iter_ancestors()
-                    if p.parent_id
-                )
-                template = 'runbot_merge.forwardport.final'
-                format_args = {
-                    'pr': new_pr,
-                    'containing': ' containing:' if ancestors else '.',
-                    'ancestors': ancestors,
-                    'footer': footer,
-                }
-            else:
-                template = 'runbot_merge.forwardport.intermediate'
-                format_args = {
-                    'pr': new_pr,
-                    'footer': footer,
-                }
-            self.env.ref(template)._send(
-                repository=new_pr.repository,
-                pull_request=new_pr.number,
-                token_field='fp_github_token',
-                format_args=format_args,
-            )
-
-            labels = ['forwardport']
-            if has_conflicts:
-                labels.append('conflict')
-            self.env['runbot_merge.pull_requests.tagging'].create({
-                'repository': new_pr.repository.id,
-                'pull_request': new_pr.number,
-                'tags_add': labels,
-            })
-
-        # try to schedule followup
-        new_batch[0]._schedule_fp_followup()
-        return new_batch.batch_id
-
     def _create_fp_branch(self, target_branch, fp_branch_name, cleanup):
         """ Creates a forward-port for the current PR to ``target_branch`` under
         ``fp_branch_name``.
@@ -969,44 +566,6 @@ stderr:
                 }
             )
 
-class Batch(models.Model):
-    _inherit = 'runbot_merge.batch'
-
-    def write(self, vals):
-        if vals.get('merge_date'):
-            self.env['forwardport.branch_remover'].create([
-                {'pr_id': p.id}
-                for b in self
-                if not b.merge_date
-                for p in b.prs
-            ])
-
-        super().write(vals)
-
-        # if we change the policy to skip CI, schedule followups on existing FPs
-        # FIXME: although this should be managed by the command handler already,
-        #        this should probably allow setting the fw policy on any batch
-        #        of the sequence
-        if vals.get('fw_policy') == 'skipci' and self.merge_date:
-            self.env['runbot_merge.pull_requests'].search([
-                ('source_id', '=', self.prs[:1].id),
-                ('state', 'not in', ('closed', 'merged')),
-            ])._schedule_fp_followup()
-
-        return True
-
-# ordering is a bit unintuitive because the lowest sequence (and name)
-# is the last link of the fp chain, reasoning is a bit more natural the
-# other way around (highest object is the last), especially with Python
-# not really having lazy sorts in the stdlib
-def branch_key(b: Branch, /, _key=itemgetter('sequence', 'name')):
-    return Reverse(_key(b))
-
-
-def pr_key(p: PullRequests, /):
-    return branch_key(p.target)
-
-
 class Stagings(models.Model):
     _inherit = 'runbot_merge.stagings'
 
@@ -1014,7 +573,7 @@ class Stagings(models.Model):
         r = super().write(vals)
         # we've just deactivated a successful staging (so it got ~merged)
         if vals.get('active') is False and self.state == 'success':
-            # check al batches to see if they should be forward ported
+            # check all batches to see if they should be forward ported
             for b in self.with_context(active_test=False).batch_ids:
                 # if all PRs of a batch have parents they're part of an FP
                 # sequence and thus handled separately, otherwise they're
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index 5d7d2313..071d2a54 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -136,7 +136,7 @@ def test_limit_after_merge(env, config, make_repo, users):
     env.run_crons()
 
     p1, p2 = env['runbot_merge.pull_requests'].search([], order='number')
-    assert p1.limit_id == p2.limit_id == branch_c, "check that limit is correctly set"
+    assert p1.limit_id == p2.limit_id == env['runbot_merge.branch'].browse(())
     pr2 = prod.get_pr(p2.number)
     with prod:
         pr1.post_comment('hansen up to b', reviewer)
diff --git a/forwardport/tests/test_overrides.py b/forwardport/tests/test_overrides.py
index eb5acf12..e41d2409 100644
--- a/forwardport/tests/test_overrides.py
+++ b/forwardport/tests/test_overrides.py
@@ -27,7 +27,7 @@ def test_override_inherited(env, config, make_repo, users):
 
     original = env['runbot_merge.pull_requests'].search([('repository.name', '=', repo.name), ('number', '=', pr.number)])
     assert original.state == 'ready'
-    assert original.limit_id.name == 'c'
+    assert not original.limit_id
 
     with repo:
         repo.post_status('staging.a', 'success')
diff --git a/forwardport/tests/test_simple.py b/forwardport/tests/test_simple.py
index cc8bb3c1..c086bd9b 100644
--- a/forwardport/tests/test_simple.py
+++ b/forwardport/tests/test_simple.py
@@ -1052,9 +1052,6 @@ class TestRecognizeCommands:
         [a] = env['runbot_merge.branch'].search([
             ('name', '=', 'a')
         ])
-        [c] = env['runbot_merge.branch'].search([
-            ('name', '=', 'c')
-        ])
 
         names = [
             "hansen",
@@ -1064,12 +1061,12 @@ class TestRecognizeCommands:
         ]
 
         for n in names:
-            assert pr_id.limit_id == c
+            assert not pr_id.limit_id
             with repo:
                 pr.post_comment(f'@{n} up to a', config['role_reviewer']['token'])
             assert pr_id.limit_id == a
             # reset state
-            pr_id.write({'limit_id': c.id})
+            pr_id.limit_id = False
 
     # FIXME: remove / merge into mergebot tests
     @pytest.mark.parametrize('indent', ['', '\N{SPACE}', '\N{SPACE}'*4, '\N{TAB}'])
@@ -1081,11 +1078,8 @@ class TestRecognizeCommands:
         [a] = env['runbot_merge.branch'].search([
             ('name', '=', 'a')
         ])
-        [c] = env['runbot_merge.branch'].search([
-            ('name', '=', 'c')
-        ])
 
-        assert pr_id.limit_id == c
+        assert not pr_id.limit_id
         with repo:
             pr.post_comment(f'{indent}@hansen up to a', config['role_reviewer']['token'])
         assert pr_id.limit_id == a
diff --git a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
index 4dd5678f..d9e64d2f 100644
--- a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
+++ b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
@@ -167,7 +167,7 @@ pr: the new forward port
 containing: label changing depending whether there are ancestors to merge
 ancestors: markdown formatted list of parent PRs which can be approved as part of the chain
 footer: a footer"
-runbot_merge.forwardport.intermediate,"This PR targets {pr.target.name} and is part of the forward-port chain. Further PRs will be created up to {pr.limit_id.name}.
+runbot_merge.forwardport.intermediate,"This PR targets {pr.target.name} and is part of the forward-port chain. Further PRs will be created up to {pr.limit_pretty}.
 {footer}","Comment when a forward port was succcessfully created but is not the last of the line.
 
 pr: the new forward port
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index 8f819349..30f69e9f 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -1,10 +1,25 @@
 from __future__ import annotations
 
+import ast
+import base64
+import contextlib
+import logging
+import os
+import re
+import typing
+from collections.abc import Iterator
+
+import requests
+
 from odoo import models, fields, api
-from odoo.tools import create_index
+from .pull_requests import PullRequests, Branch
 from .utils import enum
 
 
+_logger = logging.getLogger(__name__)
+FOOTER = '\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n'
+
+
 class Batch(models.Model):
     """ A batch is a "horizontal" grouping of *codependent* PRs: PRs with
     the same label & target but for different repositories. These are
@@ -15,8 +30,9 @@ class Batch(models.Model):
     _name = 'runbot_merge.batch'
     _description = "batch of pull request"
     _inherit = ['mail.thread']
+    _parent_store = True
 
-    target = fields.Many2one('runbot_merge.branch', required=True, index=True)
+    target = fields.Many2one('runbot_merge.branch', store=True, compute='_compute_target')
     staging_ids = fields.Many2many('runbot_merge.stagings')
     split_id = fields.Many2one('runbot_merge.split', index=True)
 
@@ -44,12 +60,45 @@ class Batch(models.Model):
         ('default', "Default"),
         ('priority', "Priority"),
         ('alone', "Alone"),
-    ], default='default', index=True, group_operator=None, required=True,
+    ], default='default', group_operator=None, required=True,
         column_type=enum(_name, 'priority'),
     )
 
     blocked = fields.Char(store=True, compute="_compute_stageable")
 
+    # unlike on PRs, this does not get detached... ? (because batches can be
+    # partially detached so that's a PR-level concern)
+    parent_path = fields.Char(index=True)
+    parent_id = fields.Many2one("runbot_merge.batch")
+    genealogy_ids = fields.Many2many(
+        "runbot_merge.batch",
+        compute="_compute_genealogy",
+        context={"active_test": False},
+    )
+
+    @property
+    def source(self):
+        return self.browse(map(int, self.parent_path.split('/', 1)[:1]))
+
+    def descendants(self, include_self: bool = False) -> Iterator[Batch]:
+        # in DB both will prefix-match on the literal prefix then apply a
+        # trivial filter (even though the filter is technically unnecessary for
+        # the first form), doing it like this means we don't have to `- self`
+        # in the `not includ_self` case
+        if include_self:
+            pattern = self.parent_path + '%'
+        else:
+            pattern = self.parent_path + '_%'
+
+        return self.search([("parent_path", '=like', pattern)])
+
+    # also depends on all the descendants of the source or sth
+    @api.depends('parent_path')
+    def _compute_genealogy(self):
+        for batch in self:
+            sid = next(iter(batch.parent_path.split('/', 1)))
+            batch.genealogy_ids = self.search([("parent_path", "=like", f"{sid}/%")], order="parent_path")
+
     def _auto_init(self):
         for field in self._fields.values():
             if not isinstance(field, fields.Selection) or field.column_type[0] == 'varchar':
@@ -65,17 +114,36 @@ class Batch(models.Model):
 
         super()._auto_init()
 
-        create_index(
-            self.env.cr,
-            "runbot_merge_batch_unblocked_idx",
-            "runbot_merge_batch",
-            ["(blocked is null), priority"],
-        )
+        self.env.cr.execute("""
+        CREATE INDEX IF NOT EXISTS runbot_merge_batch_ready_idx
+        ON runbot_merge_batch (target, priority)
+        WHERE blocked IS NULL;
+
+        CREATE INDEX IF NOT EXISTS runbot_merge_batch_parent_id_idx
+        ON runbot_merge_batch (parent_id)
+        WHERE parent_id IS NOT NULL;
+        """)
+
+    @api.depends("prs.target")
+    def _compute_target(self):
+        for batch in self:
+            if len(batch.prs) == 1:
+                batch.target = batch.prs.target
+            else:
+                targets = set(batch.prs.mapped('target'))
+                if not targets:
+                    targets = set(batch.prs.mapped('target'))
+                if len(targets) == 1:
+                    batch.target = targets.pop()
+                else:
+                    batch.target = False
+
 
     @api.depends(
         "merge_date",
         "prs.error", "prs.draft", "prs.squash", "prs.merge_method",
         "skipchecks", "prs.status", "prs.reviewed_by",
+        "prs.target"
     )
     def _compute_stageable(self):
         for batch in self:
@@ -84,9 +152,7 @@ class Batch(models.Model):
             elif blocking := batch.prs.filtered(
                 lambda p: p.error or p.draft or not (p.squash or p.merge_method)
             ):
-                batch.blocked = "Pull request(s) %s blocked." % (
-                    p.display_name for p in blocking
-                )
+                batch.blocked = "Pull request(s) %s blocked." % ', '.join(blocking.mapped('display_name'))
             elif not batch.skipchecks and (unready := batch.prs.filtered(
                 lambda p: not (p.reviewed_by and p.status == "success")
             )):
@@ -98,6 +164,8 @@ class Batch(models.Model):
                     unvalidated and f"{unvalidated} are waiting for CI",
                     failed and f"{failed} have failed CI",
                 ]))
+            elif len(targets := batch.prs.mapped('target')) > 1:
+                batch.blocked = f"Multiple target branches: {', '.join(targets.mapped('name'))!r}"
             else:
                 if batch.blocked and batch.cancel_staging:
                     batch.target.active_staging_id.cancel(
@@ -107,3 +175,333 @@ class Batch(models.Model):
                         ', '.join(batch.prs.mapped('display_name')),
                     )
                 batch.blocked = False
+
+
+    def _port_forward(self):
+        if not self:
+            return
+
+        proj = self.target.project_id
+        if not proj.fp_github_token:
+            _logger.warning(
+                "Can not forward-port %s (%s): no token on project %s",
+                self,
+                ', '.join(self.prs.mapped('display_name')),
+                proj.name
+            )
+            return
+
+        notarget = [r.name for r in self.prs.repository if not r.fp_remote_target]
+        if notarget:
+            _logger.error(
+                "Can not forward-port %s (%s): repos %s don't have a forward port remote configured",
+                self,
+                ', '.join(self.prs.mapped('display_name')),
+                ', '.join(notarget),
+            )
+            return
+
+        all_sources = [(p.source_id or p) for p in self.prs]
+        all_targets = [p._find_next_target() for p in self.prs]
+
+        if all(t is None for t in all_targets):
+            # TODO: maybe add a feedback message?
+            _logger.info(
+                "Will not forward port %s (%s): no next target",
+                self,
+                ', '.join(self.prs.mapped('display_name'))
+            )
+            return
+
+        for p, t in zip(self.prs, all_targets):
+            if t is None:
+                _logger.info("Skip forward porting %s (of %s): no next target", p.display_name, self)
+
+        # all the PRs *with a next target* should have the same, we can have PRs
+        # stopping forward port earlier but skipping... probably not
+        targets = set(filter(None, all_targets))
+        if len(targets) != 1:
+            m = dict(zip(all_targets, self.prs))
+            m.pop(None, None)
+            mm = dict(zip(self.prs, all_targets))
+            for pr in self.prs:
+                t = mm[pr]
+                # if a PR is skipped, don't flag it for discrepancy
+                if not t:
+                    continue
+
+                other, linked = next((target, p) for target, p in m.items() if target != t)
+                self.env.ref('runbot_merge.forwardport.failure.discrepancy')._send(
+                    repository=pr.repository,
+                    pull_request=pr.number,
+                    token_field='fp_github_token',
+                    format_args={'pr': pr, 'linked': linked, 'next': t.name, 'other': other.name},
+                )
+            _logger.warning(
+                "Cancelling forward-port of %s (%s): found different next branches (%s)",
+                self,
+                ', '.join(self.prs.mapped('display_name')),
+                ', '.join(t.name for t in targets),
+            )
+            return
+
+        target = targets.pop()
+        # this is run by the cron, no need to check if otherwise scheduled:
+        # either the scheduled job is this one, or it's an other scheduling
+        # which will run after this one and will see the port already exists
+        if self.search_count([('parent_id', '=', self.id), ('target', '=', target.id)]):
+            _logger.warning(
+                "Will not forward-port %s (%s): already ported",
+                self,
+                ', '.join(self.prs.mapped('display_name'))
+            )
+            return
+
+        # the base PR is the PR with the "oldest" target
+        base = max(all_sources, key=lambda p: (p.target.sequence, p.target.name))
+        # take only the branch bit
+        new_branch = '%s-%s-%s-fw' % (
+            target.name,
+            base.refname,
+            # avoid collisions between fp branches (labels can be reused
+            # or conflict especially as we're chopping off the owner)
+            base64.urlsafe_b64encode(os.urandom(3)).decode()
+        )
+        conflicts = {}
+        with contextlib.ExitStack() as s:
+            for pr in self.prs:
+                conflicts[pr], working_copy = pr._create_fp_branch(
+                    target, new_branch, s)
+
+                working_copy.push('target', new_branch)
+
+        gh = requests.Session()
+        gh.headers['Authorization'] = 'token %s' % proj.fp_github_token
+        has_conflicts = any(conflicts.values())
+        # could create a batch here but then we'd have to update `_from_gh` to
+        # take a batch and then `create` to not automatically resolve batches,
+        # easier to not do that.
+        new_batch = self.env['runbot_merge.pull_requests'].browse(())
+        self.env.cr.execute('LOCK runbot_merge_pull_requests IN SHARE MODE')
+        for pr in self.prs:
+            owner, _ = pr.repository.fp_remote_target.split('/', 1)
+            source = pr.source_id or pr
+            root = pr.root_id
+
+            message = source.message + '\n\n' + '\n'.join(
+                "Forward-Port-Of: %s" % p.display_name
+                for p in root | source
+            )
+
+            title, body = re.match(r'(?P<title>[^\n]+)\n*(?P<body>.*)', message, flags=re.DOTALL).groups()
+            r = gh.post(f'https://api.github.com/repos/{pr.repository.name}/pulls', json={
+                'base': target.name,
+                'head': f'{owner}:{new_branch}',
+                'title': '[FW]' + (' ' if title[0] != '[' else '') + title,
+                'body': body
+            })
+            if not r.ok:
+                _logger.warning("Failed to create forward-port PR for %s, deleting branches", pr.display_name)
+                # delete all the branches this should automatically close the
+                # PRs if we've created any. Using the API here is probably
+                # simpler than going through the working copies
+                for repo in self.prs.mapped('repository'):
+                    d = gh.delete(f'https://api.github.com/repos/{repo.fp_remote_target}/git/refs/heads/{new_branch}')
+                    if d.ok:
+                        _logger.info("Deleting %s:%s=success", repo.fp_remote_target, new_branch)
+                    else:
+                        _logger.warning("Deleting %s:%s=%s", repo.fp_remote_target, new_branch, d.text)
+                raise RuntimeError("Forwardport failure: %s (%s)" % (pr.display_name, r.text))
+
+            new_pr = pr._from_gh(r.json())
+            _logger.info("Created forward-port PR %s", new_pr)
+            new_batch |= new_pr
+
+            # allows PR author to close or skipci
+            new_pr.write({
+                'merge_method': pr.merge_method,
+                'source_id': source.id,
+                # only link to previous PR of sequence if cherrypick passed
+                'parent_id': pr.id if not has_conflicts else False,
+                'detach_reason': "conflicts: {}".format(
+                    f'\n{conflicts[pr]}\n{conflicts[pr]}'.strip()
+                ) if has_conflicts else None,
+            })
+            if has_conflicts and pr.parent_id and pr.state not in ('merged', 'closed'):
+                self.env.ref('runbot_merge.forwardport.failure.conflict')._send(
+                    repository=pr.repository,
+                    pull_request=pr.number,
+                    token_field='fp_github_token',
+                    format_args={'source': source, 'pr': pr, 'new': new_pr, 'footer': FOOTER},
+                )
+
+        for pr, new_pr in zip(self.prs, new_batch):
+            (h, out, err, hh) = conflicts.get(pr) or (None, None, None, None)
+
+            if h:
+                sout = serr = ''
+                if out.strip():
+                    sout = f"\nstdout:\n```\n{out}\n```\n"
+                if err.strip():
+                    serr = f"\nstderr:\n```\n{err}\n```\n"
+
+                lines = ''
+                if len(hh) > 1:
+                    lines = '\n' + ''.join(
+                        '* %s%s\n' % (sha, ' <- on this commit' if sha == h else '')
+                        for sha in hh
+                    )
+                template = 'runbot_merge.forwardport.failure'
+                format_args = {
+                    'pr': new_pr,
+                    'commits': lines,
+                    'stdout': sout,
+                    'stderr': serr,
+                    'footer': FOOTER,
+                }
+            elif has_conflicts:
+                template = 'runbot_merge.forwardport.linked'
+                format_args = {
+                    'pr': new_pr,
+                    'siblings': ', '.join(p.display_name for p in (new_batch - new_pr)),
+                    'footer': FOOTER,
+                }
+            elif not new_pr._find_next_target():
+                ancestors = "".join(
+                    f"* {p.display_name}\n"
+                    for p in pr._iter_ancestors()
+                    if p.parent_id
+                )
+                template = 'runbot_merge.forwardport.final'
+                format_args = {
+                    'pr': new_pr,
+                    'containing': ' containing:' if ancestors else '.',
+                    'ancestors': ancestors,
+                    'footer': FOOTER,
+                }
+            else:
+                template = 'runbot_merge.forwardport.intermediate'
+                format_args = {
+                    'pr': new_pr,
+                    'footer': FOOTER,
+                }
+            self.env.ref(template)._send(
+                repository=new_pr.repository,
+                pull_request=new_pr.number,
+                token_field='fp_github_token',
+                format_args=format_args,
+            )
+
+            labels = ['forwardport']
+            if has_conflicts:
+                labels.append('conflict')
+            self.env['runbot_merge.pull_requests.tagging'].create({
+                'repository': new_pr.repository.id,
+                'pull_request': new_pr.number,
+                'tags_add': labels,
+            })
+
+        new_batch = new_batch.batch_id
+        new_batch.parent_id = self
+        # try to schedule followup
+        new_batch._schedule_fp_followup()
+        return new_batch
+
+
+    def _schedule_fp_followup(self):
+        _logger = logging.getLogger(__name__).getChild('forwardport.next')
+        # if the PR has a parent and is CI-validated, enqueue the next PR
+        scheduled = self.browse(())
+        for batch in self:
+            prs = ', '.join(batch.prs.mapped('display_name'))
+            _logger.info('Checking if forward-port %s (%s)', batch, prs)
+            # in cas of conflict or update individual PRs will "lose" their
+            # parent, which should prevent forward porting
+            if not (batch.parent_id and all(p.parent_id for p in batch.prs)):
+                _logger.info('-> no parent %s (%s)', batch, prs)
+                continue
+            if not self.env.context.get('force_fw') and self.source.fw_policy != 'skipci' \
+                    and (invalid := batch.prs.filtered(lambda p: p.state not in ['validated', 'ready'])):
+                _logger.info(
+                    '-> wrong state %s (%s)',
+                    batch,
+                    ', '.join(f"{p.display_name}: {p.state}" for p in invalid),
+                )
+                continue
+
+            # check if we've already forward-ported this branch
+            next_target = self._find_next_targets()
+            if not next_target:
+                _logger.info("-> forward port done (no next target)")
+                continue
+            if len(next_target) > 1:
+                _logger.error(
+                    "-> cancelling forward-port of %s (%s): inconsistent next target branch (%s)",
+                    batch,
+                    prs,
+                    ', '.join(next_target.mapped('name')),
+                )
+
+            if n := self.search([
+                ('target', '=', next_target.id),
+                ('parent_id', '=', batch.id),
+            ], limit=1):
+                _logger.info('-> already forward-ported (%s)', n)
+                continue
+
+            _logger.info("check pending port for %s (%s)", batch, prs)
+            if self.env['forwardport.batches'].search_count([('batch_id', '=', batch.id)]):
+                _logger.warning('-> already recorded')
+                continue
+
+            _logger.info('-> ok')
+            self.env['forwardport.batches'].create({
+                'batch_id': batch.id,
+                'source': 'fp',
+            })
+            scheduled |= batch
+        return scheduled
+
+    def _find_next_target(self):
+        """Retrieves the next target from every PR, and returns it if it's the
+        same for all the PRs which have one (PRs without a next target are
+        ignored, this is considered acceptable).
+
+        If the next targets are inconsistent, returns no next target.
+        """
+        next_target = self._find_next_targets()
+        if len(next_target) == 1:
+            return next_target
+        else:
+            return self.env['runbot_merge.branch'].browse(())
+
+    def _find_next_targets(self):
+        return self.prs.mapped(lambda p: p._find_next_target() or self.env['runbot_merge.branch'])
+
+    def write(self, vals):
+        if vals.get('merge_date'):
+            # TODO: remove condition when everything is merged
+            remover = self.env.get('forwardport.branch_remover')
+            if remover is not None:
+                remover.create([
+                    {'pr_id': p.id}
+                    for b in self
+                    if not b.merge_date
+                    for p in b.prs
+                ])
+
+        if vals.get('fw_policy') == 'skipci':
+            nonskip = self.filtered(lambda b: b.fw_policy != 'skipci')
+        else:
+            nonskip = self.browse(())
+        super().write(vals)
+
+        # if we change the policy to skip CI, schedule followups on merged
+        # batches which were not previously marked as skipping CI
+        if nonskip:
+            toggled = nonskip.filtered(lambda b: b.merge_date)
+            tips = toggled.mapped(lambda b: b.genealogy_ids[-1:])
+            for tip in tips:
+                tip._schedule_fp_followup()
+
+        return True
diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index c0420f9a..b3b55a8c 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -1,12 +1,14 @@
 import logging
 import re
-from typing import List, Tuple
+from typing import List
 
 import requests
 import sentry_sdk
 
 from odoo import models, fields, api
 from odoo.exceptions import UserError
+from odoo.osv import expression
+from odoo.tools import reverse_order
 
 _logger = logging.getLogger(__name__)
 class Project(models.Model):
@@ -217,3 +219,10 @@ class Project(models.Model):
             ]
         })
         return w.action_open()
+
+    def _forward_port_ordered(self, domain=()):
+        Branches = self.env['runbot_merge.branch']
+        return Branches.search(expression.AND([
+            [('project_id', '=', self.id)],
+            domain or [],
+        ]), order=reverse_order(Branches._order))
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index f2adaa9d..ca3d90e5 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -10,14 +10,16 @@ import logging
 import re
 import time
 from functools import reduce
-from typing import Optional, Union, List, Iterator
+from operator import itemgetter
+from typing import Optional, Union, List, Iterator, Tuple
 
+import psycopg2
 import sentry_sdk
 import werkzeug
 
 from odoo import api, fields, models, tools, Command
 from odoo.osv import expression
-from odoo.tools import html_escape
+from odoo.tools import html_escape, Reverse
 from . import commands
 from .utils import enum
 
@@ -416,10 +418,6 @@ class PullRequests(models.Model):
 
     ping = fields.Char(compute='_compute_ping', recursive=True)
 
-    fw_policy = fields.Selection([
-        ('default', "Default"),
-        ('skipci', "Skip CI"),
-    ], required=True, default="default", string="Forward Port Policy")
     source_id = fields.Many2one('runbot_merge.pull_requests', index=True, help="the original source of this FP even if parents were detached along the way")
     parent_id = fields.Many2one(
         'runbot_merge.pull_requests', index=True,
@@ -515,6 +513,16 @@ class PullRequests(models.Model):
     def _linked_prs(self):
         return self.batch_id.prs - self
 
+    @property
+    def limit_pretty(self):
+        if self.limit_id:
+            return self.limit_id.name
+
+        branches = self.repository.project_id.branch_ids
+        if ((bf := self.repository.branch_filter) or '[]') != '[]':
+            branches = branches.filtered_domain(ast.literal_eval(bf))
+        return branches[:1].name
+
     @api.depends(
         'batch_id.prs.draft',
         'batch_id.prs.squash',
@@ -797,10 +805,20 @@ class PullRequests(models.Model):
                     else:
                         msg = "you can't configure forward-port CI."
                 case commands.Limit(branch) if is_author:
-                    ping, msg = self._maybe_update_limit(branch or self.target.name)
-                    if not ping:
-                        feedback(message=msg)
-                        msg = None
+                    limit = branch or self.target.name
+                    for p in self.batch_id.prs:
+                        ping, m = p._maybe_update_limit(limit)
+
+                        if ping and p == self:
+                            msg = m
+                        else:
+                            if ping:
+                                m = f"@{login} {m}"
+                            self.env['runbot_merge.pull_requests.feedback'].create({
+                                'repository': p.repository.id,
+                                'pull_request': p.number,
+                                'message': m,
+                            })
                 case commands.Limit():
                     msg = "you can't set a forward-port limit."
                 # NO!
@@ -824,6 +842,121 @@ class PullRequests(models.Model):
             feedback(message=f"@{login}{rejections}{footer}")
         return 'rejected'
 
+    def _maybe_update_limit(self, limit: str) -> Tuple[bool, str]:
+        limit_id = self.env['runbot_merge.branch'].with_context(active_test=False).search([
+            ('project_id', '=', self.repository.project_id.id),
+            ('name', '=', limit),
+        ])
+        if not limit_id:
+            return True, f"there is no branch {limit!r}, it can't be used as a forward port target."
+
+        if limit_id != self.target and not limit_id.active:
+            return True, f"branch {limit_id.name!r} is disabled, it can't be used as a forward port target."
+
+        # not forward ported yet, just acknowledge the request
+        if not self.source_id and self.state != 'merged':
+            self.limit_id = limit_id
+            if branch_key(limit_id) <= branch_key(self.target):
+                return False, "Forward-port disabled."
+            else:
+                return False, f"Forward-porting to {limit_id.name!r}."
+
+        # if the PR has been forwardported
+        prs = (self | self.forwardport_ids | self.source_id | self.source_id.forwardport_ids)
+        tip = max(prs, key=pr_key)
+        # if the fp tip was closed it's fine
+        if tip.state == 'closed':
+            return True, f"{tip.display_name} is closed, no forward porting is going on"
+
+        prs.limit_id = limit_id
+
+        real_limit = max(limit_id, tip.target, key=branch_key)
+
+        addendum = ''
+        # check if tip was queued for forward porting, try to cancel if we're
+        # supposed to stop here
+        if real_limit == tip.target and (task := self.env['forwardport.batches'].search([('batch_id', '=', tip.batch_id.id)])):
+            try:
+                with self.env.cr.savepoint():
+                    self.env.cr.execute(
+                        "SELECT FROM forwardport_batches "
+                        "WHERE id = %s FOR UPDATE NOWAIT",
+                        [task.id])
+            except psycopg2.errors.LockNotAvailable:
+                # row locked = port occurring and probably going to succeed,
+                # so next(real_limit) likely a done deal already
+                return True, (
+                    f"Forward port of {tip.display_name} likely already "
+                    f"ongoing, unable to cancel, close next forward port "
+                    f"when it completes.")
+            else:
+                self.env.cr.execute("DELETE FROM forwardport_batches WHERE id = %s", [task.id])
+
+        if real_limit != tip.target:
+            # forward porting was previously stopped at tip, and we want it to
+            # resume
+            if tip.state == 'merged':
+                self.env['forwardport.batches'].create({
+                    'batch_id': tip.batch_id.id,
+                    'source': 'fp' if tip.parent_id else 'merge',
+                })
+                resumed = tip
+            else:
+                resumed = tip.batch_id._schedule_fp_followup()
+            if resumed:
+                addendum += f', resuming forward-port stopped at {tip.display_name}'
+
+        if real_limit != limit_id:
+            addendum += f' (instead of the requested {limit_id.name!r} because {tip.display_name} already exists)'
+
+        # get a "stable" root rather than self's to avoid divertences between
+        # PRs across a root divide (where one post-root would point to the root,
+        # and one pre-root would point to the source, or a previous root)
+        root = tip.root_id
+        # reference the root being forward ported unless we are the root
+        root_ref = '' if root == self else f' {root.display_name}'
+        msg = f"Forward-porting{root_ref} to {real_limit.name!r}{addendum}."
+        # send a message to the source & root except for self, if they exist
+        root_msg = f'Forward-porting to {real_limit.name!r} (from {self.display_name}).'
+        self.env['runbot_merge.pull_requests.feedback'].create([
+            {
+                'repository': p.repository.id,
+                'pull_request': p.number,
+                'message': root_msg,
+                'token_field': 'fp_github_token',
+            }
+            # send messages to source and root unless root is self (as it
+            # already gets the normal message)
+            for p in (self.source_id | root) - self
+        ])
+
+        return False, msg
+
+
+    def _find_next_target(self) -> Optional[Branch]:
+        """ Finds the branch between target and limit_id which follows
+        reference
+        """
+        root = (self.source_id or self)
+        if self.target == root.limit_id:
+            return
+
+        branches = root.target.project_id.with_context(active_test=False)._forward_port_ordered()
+        if (branch_filter := self.repository.branch_filter) and branch_filter != '[]':
+            branches = branches.filtered_domain(ast.literal_eval(branch_filter))
+
+        branches = list(branches)
+        from_ = branches.index(self.target) + 1
+        to_ = branches.index(root.limit_id) + 1 if root.limit_id else None
+
+        # return the first active branch in the set
+        return next((
+            branch
+            for branch in branches[from_:to_]
+            if branch.active
+        ), None)
+
+
     def _approve(self, author, login):
         oldstate = self.state
         newstate = RPLUS.get(self.state)
@@ -1023,20 +1156,33 @@ class PullRequests(models.Model):
         return self.state
 
     def _get_batch(self, *, target, label):
-        Batches = self.env['runbot_merge.batch']
-        if re.search(r':patch-\d+$', label):
-            batch = Batches
-        else:
-            batch = Batches.search([
+        batch = self.env['runbot_merge.batch']
+        if not re.search(r':patch-\d+$', label):
+            batch = batch.search([
                 ('merge_date', '=', False),
-                ('target', '=', target),
+                ('prs.target', '=', target),
                 ('prs.label', '=', label),
             ])
-        return batch or Batches.create({'target': target})
+        return batch or batch.create({})
 
     @api.model
     def create(self, vals):
-        vals['batch_id'] = self._get_batch(target=vals['target'], label=vals['label']).id
+        batch = self._get_batch(target=vals['target'], label=vals['label'])
+        vals['batch_id'] = batch.id
+        if 'limit_id' not in vals:
+            limits = {p.limit_id for p in batch.prs}
+            if len(limits) == 1:
+                vals['limit_id'] = limits.pop().id
+            elif limits:
+                repo = self.env['runbot_merge.repository'].browse(vals['repository'])
+                _logger.warning(
+                    "Unable to set limit on %s#%s: found multiple limits in batch (%s)",
+                    repo.name, vals['number'],
+                    ', '.join(
+                        f'{p.display_name} => {p.limit_id.name}'
+                        for p in batch.prs
+                    )
+                )
 
         pr = super().create(vals)
         c = self.env['runbot_merge.commit'].search([('sha', '=', pr.head)])
@@ -1116,13 +1262,6 @@ class PullRequests(models.Model):
                     target=vals.get('target') or self.target.id,
                     label=vals.get('label') or self.label,
                 )
-            case None if (new_target := vals.get('target')) and new_target != self.target.id:
-                vals['batch_id'] = self._get_batch(
-                    target=new_target,
-                    label=vals.get('label') or self.label,
-                )
-                if len(self.batch_id.prs) == 1:
-                    self.env.cr.precommit.add(self.batch_id.unlink)
         w = super().write(vals)
 
         newhead = vals.get('head')
@@ -1259,6 +1398,18 @@ class PullRequests(models.Model):
             old_batch.unlink()
         return True
 
+# ordering is a bit unintuitive because the lowest sequence (and name)
+# is the last link of the fp chain, reasoning is a bit more natural the
+# other way around (highest object is the last), especially with Python
+# not really having lazy sorts in the stdlib
+def branch_key(b: Branch, /, _key=itemgetter('sequence', 'name')):
+    return Reverse(_key(b))
+
+
+def pr_key(p: PullRequests, /):
+    return branch_key(p.target)
+
+
 # state changes on reviews
 RPLUS = {
     'opened': 'approved',
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index f4a65b61..46e87ab8 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -158,7 +158,7 @@
                             <field name="merge_method"/>
                             <field name="squash"/>
                             <field name="draft"/>
-                            <field string="Up To" name="limit_id"/>
+<!--                            <field string="Up To" name="limit_id"/>-->
 <!--                            <field string="Forward-Port Policy" name="fw_policy"/>-->
                         </group>
                     </group>

From 124d1212a2c28c7cd36bb7bfaf8af71974ec9a72 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 20 Feb 2024 11:24:35 +0100
Subject: [PATCH 095/192] [ADD] forwardport: tests that fw batches can vary

This tests that the new setup *does* allow both removing PRs from a
forward-ported batch (something which may have worked previously
already, anyway) and importantly *adding* PRs to a forward ported
batch.

The updated batch behaves somewhat like a new batch, but it should
retain the history via linking through the batch, and it allows
cross-repo fixes which were not necessary earlier (e.g. because we're
touching an API of repo A which was not used in repo B in earlier
branches, but now is), something which was not really possible before
the refactoring of batches & co.
---
 forwardport/tests/test_batches.py | 126 +++++++++++++++++++++++++++++-
 1 file changed, 125 insertions(+), 1 deletion(-)

diff --git a/forwardport/tests/test_batches.py b/forwardport/tests/test_batches.py
index a637e7a0..4c72f3da 100644
--- a/forwardport/tests/test_batches.py
+++ b/forwardport/tests/test_batches.py
@@ -1,4 +1,4 @@
-from utils import Commit, make_basic
+from utils import Commit, make_basic, to_pr
 
 
 def test_single_updated(env, config, make_repo):
@@ -87,3 +87,127 @@ def test_single_updated(env, config, make_repo):
 
     assert pr22_id.source_id == pr2_id
     assert pr22_id.parent_id == pr21_id
+
+def test_closing_during_fp(env, config, make_repo, users):
+    """ Closing a PR after it's been ported once should not port it further, but
+    the rest of the batch should carry on
+    """
+    r1, _ = make_basic(env, config, make_repo)
+    r2, _ = make_basic(env, config, make_repo)
+    env['runbot_merge.repository'].search([]).required_statuses = 'default'
+
+    with r1, r2:
+        r1.make_commits('a', Commit('1', tree={'1': '0'}), ref='heads/aref')
+        pr1 = r1.make_pr(target='a', head='aref')
+        r1.post_status('aref', 'success')
+        pr1.post_comment('hansen r+', config['role_reviewer']['token'])
+
+        r2.make_commits('a', Commit('2', tree={'2': '0'}), ref='heads/aref')
+        pr2 = r2.make_pr(target='a', head='aref')
+        r2.post_status('aref', 'success')
+        pr2.post_comment('hansen r+', config['role_reviewer']['token'])
+    env.run_crons()
+
+    with r1, r2:
+        r1.post_status('staging.a', 'success')
+        r2.post_status('staging.a', 'success')
+    env.run_crons()
+
+    pr1_id = to_pr(env, pr1)
+    [pr1_1_id] = pr1_id.forwardport_ids
+    pr2_id = to_pr(env, pr2)
+    [pr2_1_id] = pr2_id.forwardport_ids
+
+    with r1:
+        r1.get_pr(pr1_1_id.number).close(config['role_user']['token'])
+
+    with r2:
+        r2.post_status(pr2_1_id.head, 'success')
+    env.run_crons()
+
+    assert env['runbot_merge.pull_requests'].search_count([]) == 5,\
+        "only one of the forward ports should be ported"
+    assert not env['runbot_merge.pull_requests'].search([('parent_id', '=', pr1_1_id.id)]),\
+        "the closed PR should not be ported"
+
+def test_add_pr_during_fp(env, config, make_repo, users):
+    """ It should be possible to add new PRs to an FP batch
+    """
+    r1, _ = make_basic(env, config, make_repo)
+    r2, fork2 = make_basic(env, config, make_repo)
+    repos = env['runbot_merge.repository'].search([])
+    repos.required_statuses = 'default'
+    # needs a "d" branch
+    repos[0].project_id.write({
+        'branch_ids': [(0, 0, {'name': 'd', 'sequence': 40})],
+    })
+    with r1, r2:
+        r1.make_ref("heads/d", r1.commit("c").id)
+        r2.make_ref("heads/d", r2.commit("c").id)
+
+    with r1, r2:
+        r1.make_commits('a', Commit('1', tree={'1': '0'}), ref='heads/aref')
+        pr1_a = r1.make_pr(target='a', head='aref')
+        r1.post_status('aref', 'success')
+        pr1_a.post_comment('hansen r+', config['role_reviewer']['token'])
+    env.run_crons()
+
+    with r1, r2:
+        r1.post_status('staging.a', 'success')
+        r2.post_status('staging.a', 'success')
+    env.run_crons()
+
+    pr1_a_id = to_pr(env, pr1_a)
+    [pr1_b_id] = pr1_a_id.forwardport_ids
+
+    with r2, fork2:
+        fork2.make_commits('b', Commit('2', tree={'2': '0'}), ref=f'heads/{pr1_b_id.refname}')
+        pr2_b = r2.make_pr(title="B", target='b', head=f'{fork2.owner}:{pr1_b_id.refname}')
+    env.run_crons()
+
+    pr2_b_id = to_pr(env, pr2_b)
+
+    assert not pr1_b_id.staging_id
+    assert not pr2_b_id.staging_id
+    assert pr1_b_id.batch_id == pr2_b_id.batch_id
+    assert pr1_b_id.state == "opened",\
+        "implicit approval from forward port should have been canceled"
+    batch = pr2_b_id.batch_id
+
+    with r1:
+        r1.post_status(pr1_b_id.head, 'success')
+        r1.get_pr(pr1_b_id.number).post_comment('hansen r+', config['role_reviewer']['token'])
+    env.run_crons()
+
+    assert batch.blocked
+    assert pr1_b_id.blocked
+
+    with r2:
+        r2.post_status(pr2_b.head, "success")
+        pr2_b.post_comment("hansen r+", config['role_reviewer']['token'])
+    env.run_crons()
+
+    assert not batch.blocked
+    assert pr1_b_id.staging_id and pr1_b_id.staging_id == pr2_b_id.staging_id
+
+    with r1, r2:
+        r1.post_status('staging.b', 'success')
+        r2.post_status('staging.b', 'success')
+    env.run_crons()
+
+    def find_child(pr):
+        return env['runbot_merge.pull_requests'].search([
+            ('parent_id', '=', pr.id),
+        ])
+    pr1_c_id = find_child(pr1_b_id)
+    assert pr1_c_id
+    pr2_c_id = find_child(pr2_b_id)
+    assert pr2_c_id
+
+    with r1, r2:
+        r1.post_status(pr1_c_id.head, 'success')
+        r2.post_status(pr2_c_id.head, 'success')
+    env.run_crons()
+
+    assert find_child(pr1_c_id)
+    assert find_child(pr2_c_id)

From 94fe0329b41cb59022ef68430399abd3cccad371 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 23 Feb 2024 13:46:37 +0100
Subject: [PATCH 096/192] [FIX] *: behaviour around branch deactivation & fw
 maintenance

Test and refine the handling of batch forward ports around branch
deactivation, especially with differential. Notably, fix an error in
the conversion of the FW process to batches: individual PR limit was
not correctly taken in account during forward port unless *all* PRs
were done, even though that is a primary motivation for the
change.

Partial forward porting should now work correctly, and the detection
and handling of differential next target should be better handled to
boot.

Significantly rework the interplay between batches and PRs being
closed in order to maintain sequencing / consistency of forward port
sequences: previously a batch would get deleted if all its PRs are
closed, but that is an issue when it is part of a forward port
sequence as we now lose information.

Instead, detach the PRs from the batch as before but have the batch
skip unlinking if it has historical value (parent or child
batch). Currently the batch's state is a bit weird as it doesn't get
merged, but...

While at it, significantly simplify `_try_closing` as it turns out to
have a ton of incidental / historical complexity from old attempts at
fixing concurrency issues, which should not be necessary anymore and
in fact actively interfere with the new and more compute-heavy state
of things.
---
 forwardport/models/project.py        |  45 +++--
 forwardport/tests/test_weird.py      | 250 +++++++++++++++++++++++++++
 runbot_merge/models/batch.py         |  81 +++++----
 runbot_merge/models/pull_requests.py |  36 ++--
 4 files changed, 338 insertions(+), 74 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 137e69ea..9fb62a6e 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -26,6 +26,7 @@ import requests
 
 from odoo import models, fields, api
 from odoo.exceptions import UserError
+from odoo.osv import expression
 from odoo.tools.misc import topological_sort, groupby
 from odoo.tools.appdirs import user_cache_dir
 from odoo.addons.base.models.res_partner import Partner
@@ -62,35 +63,43 @@ class Project(models.Model):
         originally disabled (and thus skipped over)
         """
         Batch = self.env['runbot_merge.batch']
+        ported = self.env['runbot_merge.pull_requests']
         for p in self:
             actives = previously_active_branches[p]
             for deactivated in p.branch_ids.filtered(lambda b: not b.active) & actives:
-                # if a PR targets a deactivated branch, and that's not its limit,
-                # and it doesn't have a child (e.g. CI failed), enqueue a forward
-                # port as if the now deactivated branch had been skipped over (which
-                # is the normal fw behaviour)
+                # if a non-merged batch targets a deactivated branch which is
+                # not its limit
                 extant = Batch.search([
+                    ('parent_id', '!=', False),
                     ('target', '=', deactivated.id),
-                    # FIXME: this should probably be *all* the PRs of the batch
+                    # if at least one of the PRs has a different limit
                     ('prs.limit_id', '!=', deactivated.id),
                     ('merge_date', '=', False),
-                ])
+                ]).filtered(lambda b:\
+                    # and has a next target (should already be a function of
+                    # the search but doesn't hurt)
+                    b._find_next_target() \
+                    # and has not already been forward ported
+                    and Batch.search_count([('parent_id', '=', b.id)]) == 0
+                )
+                ported |= extant.prs.filtered(lambda p: p._find_next_target())
+                # enqueue a forward port as if the now deactivated branch had
+                # been skipped over (which is the normal fw behaviour)
                 for b in extant.with_context(force_fw=True):
-                    next_target = b._find_next_target()
-                    # should not happen since we already filtered out limits
-                    if not next_target:
-                        continue
-
-                    # check if it has a descendant in the next branch, if so skip
-                    if Batch.search_count([
-                        ('parent_id', '=', b.id),
-                        ('target', '=', next_target.id)
-                    ]):
-                        continue
-
                     # otherwise enqueue a followup
                     b._schedule_fp_followup()
 
+        if not ported:
+            return
+
+        for feedback in self.env['runbot_merge.pull_requests.feedback'].search(expression.OR(
+            [('repository', '=', p.repository.id), ('pull_request', '=', p.number)]
+            for p in ported
+        )):
+            # FIXME: better signal
+            if 'disabled' in feedback.message:
+                feedback.message += '\n\nAs this was not its limit, it will automatically be forward ported to the next active branch.'
+
     def _insert_intermediate_prs(self, branches_before):
         """If new branches have been added to the sequence inbetween existing
         branches (mostly a freeze inserted before the main branch), fill in
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index 0c2170b4..b97db3c0 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -918,3 +918,253 @@ def test_missing_magic_ref(env, config, make_repo):
     # what they are (rather than e.g. diff the HEAD it branch with the target)
     # as a result it doesn't forwardport our fake, we'd have to reset the PR's
     # branch for that to happen
+
+def test_disable_branch_with_batches(env, config, make_repo, users):
+    """We want to avoid losing pull requests, so when deactivating a branch,
+    if there are *forward port* batches targeting that branch which have not
+    been forward ported yet port them over, as if their source had been merged
+    after the branch was disabled (thus skipped over)
+    """
+    proj, repo, fork = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
+    proj.repo_ids.required_statuses = "default"
+    branch_b = env['runbot_merge.branch'].search([('name', '=', 'b')])
+    assert branch_b
+
+    # region repo2 creation & setup
+    repo2 = make_repo('proj2')
+    with repo2:
+        [a, b, c] = repo2.make_commits(
+            None,
+            Commit("a", tree={"f": "a"}),
+            Commit("b", tree={"g": "b"}),
+            Commit("c", tree={"h": "c"}),
+        )
+        repo2.make_ref("heads/a", a)
+        repo2.make_ref("heads/b", b)
+        repo2.make_ref("heads/c", c)
+    fork2 = repo2.fork()
+    repo2_id = env['runbot_merge.repository'].create({
+        "project_id": proj.id,
+        "name": repo2.name,
+        "required_statuses": "default",
+        "fp_remote_target": fork2.name,
+    })
+    env['res.partner'].search([
+        ('github_login', '=', config['role_reviewer']['user'])
+    ]).write({
+        'review_rights': [(0, 0, {'repository_id': repo2_id.id, 'review': True})]
+    })
+    env['res.partner'].search([
+        ('github_login', '=', config['role_self_reviewer']['user'])
+    ]).write({
+        'review_rights': [(0, 0, {'repository_id': repo2_id.id, 'self_review': True})]
+    })
+    # endregion
+
+    # region set up forward ported batch
+    with repo, fork, repo2, fork2:
+        fork.make_commits("a", Commit("x", tree={"x": "1"}), ref="heads/x")
+        pr1_a = repo.make_pr(title="X", target="a", head=f"{fork.owner}:x")
+        pr1_a.post_comment("hansen r+", config['role_reviewer']['token'])
+        repo.post_status(pr1_a.head, "success")
+
+        fork2.make_commits("a", Commit("x", tree={"x": "1"}), ref="heads/x")
+        pr2_a = repo2.make_pr(title="X", target="a", head=f"{fork2.owner}:x")
+        pr2_a.post_comment("hansen r+", config['role_reviewer']['token'])
+        repo2.post_status(pr2_a.head, "success")
+    # remove just pr2 from the forward ports (maybe?)
+    pr2_a_id = to_pr(env, pr2_a)
+    pr2_a_id.limit_id = branch_b.id
+    env.run_crons()
+    assert pr2_a_id.limit_id == branch_b
+    # endregion
+
+
+    with repo, repo2:
+        repo.post_status('staging.a', 'success')
+        repo2.post_status('staging.a', 'success')
+    env.run_crons()
+
+    PullRequests = env['runbot_merge.pull_requests']
+    pr1_b_id = PullRequests.search([('parent_id', '=', to_pr(env, pr1_a).id)])
+    pr2_b_id = PullRequests.search([('parent_id', '=', pr2_a_id.id)])
+    assert pr1_b_id.parent_id
+    assert pr1_b_id.state == 'opened'
+    assert pr2_b_id.parent_id
+    assert pr2_b_id.state == 'opened'
+
+    b_id = proj.branch_ids.filtered(lambda b: b.name == 'b')
+    proj.write({
+        'branch_ids': [(1, b_id.id, {'active': False})]
+    })
+    env.run_crons()
+    assert not b_id.active
+    assert PullRequests.search_count([]) == 5, "should have ported pr1 but not pr2"
+    assert PullRequests.search([], order="number DESC", limit=1).parent_id == pr1_b_id
+
+    assert repo.get_pr(pr1_b_id.number).comments == [
+        seen(env, repo.get_pr(pr1_b_id.number), users),
+        (users['user'], "This PR targets b and is part of the forward-port chain. Further PRs will be created up to c.\n\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n"),
+        (users['user'], "@{user} @{reviewer} the target branch 'b' has been disabled, you may want to close this PR.\n\nAs this was not its limit, it will automatically be forward ported to the next active branch.".format_map(users)),
+    ]
+    assert repo2.get_pr(pr2_b_id.number).comments == [
+        seen(env, repo2.get_pr(pr2_b_id.number), users),
+        (users['user'], """\
+@{user} @{reviewer} this PR targets b and is the last of the forward-port chain.
+
+To merge the full chain, use
+> @hansen r+
+
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+""".format_map(users)),
+        (users['user'], "@{user} @{reviewer} the target branch 'b' has been disabled, you may want to close this PR.".format_map(users)),
+    ]
+
+def test_disable_multitudes(env, config, make_repo, users, setreviewers):
+    """Ensure that deactivation ports can jump over other deactivated branches.
+    """
+    # region setup
+    repo = make_repo("bob")
+    project = env['runbot_merge.project'].create({
+        "name": "bob",
+        "github_token": config['github']['token'],
+        "github_prefix": "hansen",
+        "fp_github_token": config['github']['token'],
+        "fp_github_name": "herbert",
+        "branch_ids": [
+            (0, 0, {'name': 'a', 'sequence': 90}),
+            (0, 0, {'name': 'b', 'sequence': 80}),
+            (0, 0, {'name': 'c', 'sequence': 70}),
+            (0, 0, {'name': 'd', 'sequence': 60}),
+        ],
+        "repo_ids": [(0, 0, {
+            'name': repo.name,
+            'required_statuses': 'default',
+            'fp_remote_target': repo.name,
+        })],
+    })
+    setreviewers(project.repo_ids)
+
+    with repo:
+        [a, b, c, d] = repo.make_commits(
+            None,
+            Commit("a", tree={"branch": "a"}),
+            Commit("b", tree={"branch": "b"}),
+            Commit("c", tree={"branch": "c"}),
+            Commit("d", tree={"branch": "d"}),
+        )
+        repo.make_ref("heads/a", a)
+        repo.make_ref("heads/b", b)
+        repo.make_ref("heads/c", c)
+        repo.make_ref("heads/d", d)
+    # endregion
+
+    with repo:
+        [a] = repo.make_commits("a", Commit("X", tree={"x": "1"}), ref="heads/x")
+        pra = repo.make_pr(target="a", head="x")
+        pra.post_comment("hansen r+", config['role_reviewer']['token'])
+        repo.post_status(a, "success")
+    env.run_crons()
+
+    with repo:
+        repo.post_status('staging.a', 'success')
+    env.run_crons()
+
+    pra_id = to_pr(env, pra)
+    assert pra_id.state == 'merged'
+
+    prb_id = env['runbot_merge.pull_requests'].search([('target.name', '=', 'b')])
+    assert prb_id.parent_id == pra_id
+
+    project.write({
+        'branch_ids': [
+            (1, b.id, {'active': False})
+            for b in env['runbot_merge.branch'].search([('name', 'in', ['b', 'c'])])
+        ]
+    })
+    env.run_crons()
+
+    # should not have ported prb to the disabled branch c
+    assert not env['runbot_merge.pull_requests'].search([('target.name', '=', 'c')])
+
+    # should have ported prb to the active branch d
+    prd_id = env['runbot_merge.pull_requests'].search([('target.name', '=', 'd')])
+    assert prd_id
+    assert prd_id.parent_id == prb_id
+
+    prb = repo.get_pr(prb_id.number)
+    assert prb.comments == [
+        seen(env, prb, users),
+        (users['user'], 'This PR targets b and is part of the forward-port chain. Further PRs will be created up to d.\n\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n'),
+        (users['user'], """\
+@{user} @{reviewer} the target branch 'b' has been disabled, you may want to close this PR.
+
+As this was not its limit, it will automatically be forward ported to the next active branch.\
+""".format_map(users)),
+    ]
+    prd = repo.get_pr(prd_id.number)
+    assert prd.comments == [
+        seen(env, prd, users),
+        (users['user'], """\
+@{user} @{reviewer} this PR targets d and is the last of the forward-port chain.
+
+To merge the full chain, use
+> @hansen r+
+
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+""".format_map(users))
+    ]
+
+def test_maintain_batch_history(env, config, make_repo, users):
+    """Batches which are part of a forward port sequence should not be deleted
+    even if all their PRs are closed.
+
+    Sadly in that case it's a bit difficult to maintain the integrity of the
+    batch as each PR being closed (until the last one?) will be removed from
+    the batch.
+    """
+    proj, repo, fork = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
+    proj.repo_ids.required_statuses = "default"
+
+    with repo, fork:
+        fork.make_commits("a", Commit("x", tree={"x": "1"}), ref="heads/x")
+        pr1_a = repo.make_pr(title="X", target="a", head=f"{fork.owner}:x")
+        pr1_a.post_comment("hansen r+", config['role_reviewer']['token'])
+        repo.post_status(pr1_a.head, "success")
+    env.run_crons()
+
+    pr1_a_id = to_pr(env, pr1_a)
+    with repo:
+        repo.post_status('staging.a', 'success')
+    env.run_crons()
+
+    pr1_b_id = env['runbot_merge.pull_requests'].search([('parent_id', '=', pr1_a_id.id)])
+    with repo:
+        repo.post_status(pr1_b_id.head, 'success')
+    env.run_crons()
+
+    pr1_c_id = env['runbot_merge.pull_requests'].search([('parent_id', '=', pr1_b_id.id)])
+
+    # region check that all the batches are set up correctly
+    assert pr1_a_id.batch_id
+    assert pr1_b_id.batch_id
+    assert pr1_c_id.batch_id
+    assert pr1_c_id.batch_id.parent_id == pr1_b_id.batch_id
+    assert pr1_b_id.batch_id.parent_id == pr1_a_id.batch_id
+    b_batch = pr1_b_id.batch_id
+    # endregion
+
+    pr1_b = repo.get_pr(pr1_b_id.number)
+    with repo:
+        pr1_b.close()
+    env.run_crons()
+    assert pr1_b_id.state == 'closed'
+
+    # region check that all the batches are *still* set up correctly
+    assert pr1_a_id.batch_id
+    assert not pr1_b_id.batch_id, "the PR still gets detached from the batch"
+    assert b_batch.exists(), "the batch is not deleted tho"
+    assert pr1_c_id.batch_id
+    assert pr1_c_id.batch_id.parent_id == b_batch
+    assert b_batch.parent_id == pr1_a_id.batch_id
+    # endregion
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index 30f69e9f..e689468b 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -1,18 +1,16 @@
 from __future__ import annotations
 
-import ast
 import base64
 import contextlib
 import logging
 import os
 import re
-import typing
+from collections import defaultdict
 from collections.abc import Iterator
 
 import requests
 
 from odoo import models, fields, api
-from .pull_requests import PullRequests, Branch
 from .utils import enum
 
 
@@ -142,8 +140,8 @@ class Batch(models.Model):
     @api.depends(
         "merge_date",
         "prs.error", "prs.draft", "prs.squash", "prs.merge_method",
-        "skipchecks", "prs.status", "prs.reviewed_by",
-        "prs.target"
+        "skipchecks",
+        "prs.status", "prs.reviewed_by", "prs.target",
     )
     def _compute_stageable(self):
         for batch in self:
@@ -213,30 +211,32 @@ class Batch(models.Model):
             )
             return
 
+        PRs = self.env['runbot_merge.pull_requests']
+        targets = defaultdict(lambda: PRs)
         for p, t in zip(self.prs, all_targets):
-            if t is None:
+            if t:
+                targets[t] |= p
+            else:
                 _logger.info("Skip forward porting %s (of %s): no next target", p.display_name, self)
 
+
         # all the PRs *with a next target* should have the same, we can have PRs
         # stopping forward port earlier but skipping... probably not
-        targets = set(filter(None, all_targets))
         if len(targets) != 1:
-            m = dict(zip(all_targets, self.prs))
-            m.pop(None, None)
-            mm = dict(zip(self.prs, all_targets))
-            for pr in self.prs:
-                t = mm[pr]
-                # if a PR is skipped, don't flag it for discrepancy
-                if not t:
-                    continue
-
-                other, linked = next((target, p) for target, p in m.items() if target != t)
-                self.env.ref('runbot_merge.forwardport.failure.discrepancy')._send(
-                    repository=pr.repository,
-                    pull_request=pr.number,
-                    token_field='fp_github_token',
-                    format_args={'pr': pr, 'linked': linked, 'next': t.name, 'other': other.name},
-                )
+            for t, prs in targets.items():
+                linked, other = next((
+                    (linked, other)
+                    for other, linkeds in targets.items()
+                    if other != t
+                    for linked in linkeds
+                ))
+                for pr in prs:
+                    self.env.ref('runbot_merge.forwardport.failure.discrepancy')._send(
+                        repository=pr.repository,
+                        pull_request=pr.number,
+                        token_field='fp_github_token',
+                        format_args={'pr': pr, 'linked': linked, 'next': t.name, 'other': other.name},
+                    )
             _logger.warning(
                 "Cancelling forward-port of %s (%s): found different next branches (%s)",
                 self,
@@ -245,7 +245,7 @@ class Batch(models.Model):
             )
             return
 
-        target = targets.pop()
+        target, prs = next(iter(targets.items()))
         # this is run by the cron, no need to check if otherwise scheduled:
         # either the scheduled job is this one, or it's an other scheduling
         # which will run after this one and will see the port already exists
@@ -253,7 +253,7 @@ class Batch(models.Model):
             _logger.warning(
                 "Will not forward-port %s (%s): already ported",
                 self,
-                ', '.join(self.prs.mapped('display_name'))
+                ', '.join(prs.mapped('display_name'))
             )
             return
 
@@ -269,7 +269,7 @@ class Batch(models.Model):
         )
         conflicts = {}
         with contextlib.ExitStack() as s:
-            for pr in self.prs:
+            for pr in prs:
                 conflicts[pr], working_copy = pr._create_fp_branch(
                     target, new_branch, s)
 
@@ -281,9 +281,9 @@ class Batch(models.Model):
         # could create a batch here but then we'd have to update `_from_gh` to
         # take a batch and then `create` to not automatically resolve batches,
         # easier to not do that.
-        new_batch = self.env['runbot_merge.pull_requests'].browse(())
+        new_batch = PRs.browse(())
         self.env.cr.execute('LOCK runbot_merge_pull_requests IN SHARE MODE')
-        for pr in self.prs:
+        for pr in prs:
             owner, _ = pr.repository.fp_remote_target.split('/', 1)
             source = pr.source_id or pr
             root = pr.root_id
@@ -305,15 +305,15 @@ class Batch(models.Model):
                 # delete all the branches this should automatically close the
                 # PRs if we've created any. Using the API here is probably
                 # simpler than going through the working copies
-                for repo in self.prs.mapped('repository'):
+                for repo in prs.mapped('repository'):
                     d = gh.delete(f'https://api.github.com/repos/{repo.fp_remote_target}/git/refs/heads/{new_branch}')
                     if d.ok:
                         _logger.info("Deleting %s:%s=success", repo.fp_remote_target, new_branch)
                     else:
                         _logger.warning("Deleting %s:%s=%s", repo.fp_remote_target, new_branch, d.text)
-                raise RuntimeError("Forwardport failure: %s (%s)" % (pr.display_name, r.text))
+                raise RuntimeError(f"Forwardport failure: {pr.display_name} ({r.text})")
 
-            new_pr = pr._from_gh(r.json())
+            new_pr = PRs._from_gh(r.json())
             _logger.info("Created forward-port PR %s", new_pr)
             new_batch |= new_pr
 
@@ -335,7 +335,7 @@ class Batch(models.Model):
                     format_args={'source': source, 'pr': pr, 'new': new_pr, 'footer': FOOTER},
                 )
 
-        for pr, new_pr in zip(self.prs, new_batch):
+        for pr, new_pr in zip(prs, new_batch):
             (h, out, err, hh) = conflicts.get(pr) or (None, None, None, None)
 
             if h:
@@ -371,6 +371,8 @@ class Batch(models.Model):
                     f"* {p.display_name}\n"
                     for p in pr._iter_ancestors()
                     if p.parent_id
+                    if p.state not in ('closed', 'merged')
+                    if p.target.active
                 )
                 template = 'runbot_merge.forwardport.final'
                 format_args = {
@@ -505,3 +507,18 @@ class Batch(models.Model):
                 tip._schedule_fp_followup()
 
         return True
+
+    def unlink(self):
+        """
+        batches can be unlinked if they:
+
+        - have run out of PRs
+        - and don't have a parent batch (which is not being deleted)
+        - and don't have a child batch (which is not being deleted)
+
+        this is to keep track of forward port histories at the batch level
+        """
+        unlinkable = self.filtered(
+            lambda b: not (b.prs or (b.parent_id - self) or (self.search([('parent_id', '=', b.id)]) - self))
+        )
+        return super(Batch, unlinkable).unlink()
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index ca3d90e5..ee6b76e5 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -288,9 +288,8 @@ class Branch(models.Model):
             b.display_name += ' (inactive)'
 
     def write(self, vals):
-        super().write(vals)
-        if vals.get('active') is False:
-            self.active_staging_id.cancel(
+        if vals.get('active') is False and (actives := self.filtered('active')):
+            actives.active_staging_id.cancel(
                 "Target branch deactivated by %r.",
                 self.env.user.login,
             )
@@ -299,8 +298,9 @@ class Branch(models.Model):
                 'repository': pr.repository.id,
                 'pull_request': pr.number,
                 'message': tmpl._format(pr=pr),
-            } for pr in self.prs])
+            } for pr in actives.prs])
             self.env.ref('runbot_merge.branch_cleanup')._trigger()
+        super().write(vals)
         return True
 
     @api.depends('staging_ids.active')
@@ -939,7 +939,7 @@ class PullRequests(models.Model):
         """
         root = (self.source_id or self)
         if self.target == root.limit_id:
-            return
+            return None
 
         branches = root.target.project_id.with_context(active_test=False)._forward_port_ordered()
         if (branch_filter := self.repository.branch_filter) and branch_filter != '[]':
@@ -1255,8 +1255,7 @@ class PullRequests(models.Model):
             case True if not self.closed:
                 vals['reviewed_by'] = False
                 vals['batch_id'] = False
-                if len(self.batch_id.prs) == 1:
-                    self.env.cr.precommit.add(self.batch_id.unlink)
+                self.env.cr.precommit.add(self.batch_id.unlink)
             case False if self.closed:
                 vals['batch_id'] = self._get_batch(
                     target=vals.get('target') or self.target.id,
@@ -1373,29 +1372,18 @@ class PullRequests(models.Model):
         # ignore if the PR is already being updated in a separate transaction
         # (most likely being merged?)
         self.env.cr.execute('''
-        SELECT id, state, batch_id FROM runbot_merge_pull_requests
-        WHERE id = %s AND state != 'merged'
+        SELECT batch_id FROM runbot_merge_pull_requests
+        WHERE id = %s AND state != 'merged' AND state != 'closed'
         FOR UPDATE SKIP LOCKED;
         ''', [self.id])
         r = self.env.cr.fetchone()
         if not r:
             return False
 
-        self.env.cr.execute('''
-        UPDATE runbot_merge_pull_requests
-        SET closed=True, reviewed_by = null
-        WHERE id = %s
-        ''', [self.id])
-        self.env.cr.commit()
-        self.modified(['closed', 'reviewed_by', 'batch_id'])
         self.unstage("closed by %s", by)
+        # `write` automatically unsets reviewer & batch when closing but...
+        self.write({'closed': True, 'reviewed_by': False, 'batch_id': False})
 
-        # PRs should leave their batch when *closed*, and batches must die when
-        # no PR is linked
-        old_batch = self.batch_id
-        self.batch_id = False
-        if not old_batch.prs:
-            old_batch.unlink()
         return True
 
 # ordering is a bit unintuitive because the lowest sequence (and name)
@@ -1507,10 +1495,10 @@ class Feedback(models.Model):
     """
     _name = _description = 'runbot_merge.pull_requests.feedback'
 
-    repository = fields.Many2one('runbot_merge.repository', required=True)
+    repository = fields.Many2one('runbot_merge.repository', required=True, index=True)
     # store the PR number (not id) as we may want to send feedback to PR
     # objects on non-handled branches
-    pull_request = fields.Integer(group_operator=None)
+    pull_request = fields.Integer(group_operator=None, index=True)
     message = fields.Char()
     close = fields.Boolean()
     token_field = fields.Selection(

From 6972749fee3ffcbf8152682b505474c285525bed Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 26 Feb 2024 09:51:22 +0100
Subject: [PATCH 097/192] [IMP] forwardport: remove duplicate-ish make_basic in
 test_weird

The one from the utils can easily be adapted for the job, and
documented while at it, no need for an extra 50 lines or so of
complete duplicate.
---
 forwardport/tests/test_weird.py | 109 ++++++--------------------------
 mergebot_test_utils/utils.py    |  57 +++++++++++++----
 2 files changed, 67 insertions(+), 99 deletions(-)

diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index b97db3c0..704d5c2a 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -3,84 +3,15 @@ from datetime import datetime
 
 import pytest
 
-from utils import seen, Commit, to_pr
+from utils import seen, Commit, to_pr, make_basic
 
 
-def make_basic(env, config, make_repo, *, fp_token, fp_remote):
-    """ Creates a basic repo with 3 forking branches
-
-    0 -- 1 -- 2 -- 3 -- 4  : a
-              |
-              `-- 11 -- 22 : b
-                  |
-                  `-- 111  : c
-    each branch just adds and modifies a file (resp. f, g and h) through the
-    contents sequence a b c d e
-    """
-    Projects = env['runbot_merge.project']
-    project = Projects.search([('name', '=', 'myproject')])
-    if not project:
-        project = Projects.create({
-            'name': 'myproject',
-            'github_token': config['github']['token'],
-            'github_prefix': 'hansen',
-            'fp_github_token': fp_token and config['github']['token'],
-            'fp_github_name': 'herbert',
-            'branch_ids': [
-                (0, 0, {'name': 'a', 'sequence': 2}),
-                (0, 0, {'name': 'b', 'sequence': 1}),
-                (0, 0, {'name': 'c', 'sequence': 0}),
-            ],
-        })
-
-    prod = make_repo('proj')
-    with prod:
-        a_0, a_1, a_2, a_3, a_4, = prod.make_commits(
-            None,
-            Commit("0", tree={'f': 'a'}),
-            Commit("1", tree={'f': 'b'}),
-            Commit("2", tree={'f': 'c'}),
-            Commit("3", tree={'f': 'd'}),
-            Commit("4", tree={'f': 'e'}),
-            ref='heads/a',
-        )
-        b_1, b_2 = prod.make_commits(
-            a_2,
-            Commit('11', tree={'g': 'a'}),
-            Commit('22', tree={'g': 'b'}),
-            ref='heads/b',
-        )
-        prod.make_commits(
-            b_1,
-            Commit('111', tree={'h': 'a'}),
-            ref='heads/c',
-        )
-    other = prod.fork()
-    repo = env['runbot_merge.repository'].create({
-        'project_id': project.id,
-        'name': prod.name,
-        'required_statuses': 'legal/cla,ci/runbot',
-        'fp_remote_target': fp_remote and other.name,
-    })
-    env['res.partner'].search([
-        ('github_login', '=', config['role_reviewer']['user'])
-    ]).write({
-        'review_rights': [(0, 0, {'repository_id': repo.id, 'review': True})]
-    })
-    env['res.partner'].search([
-        ('github_login', '=', config['role_self_reviewer']['user'])
-    ]).write({
-        'review_rights': [(0, 0, {'repository_id': repo.id, 'self_review': True})]
-    })
-
-    return project, prod, other
-
 def test_no_token(env, config, make_repo):
     """ if there's no token on the repo, nothing should break though should
     log
     """
     # create project configured with remotes on the repo but no token
-    proj, prod, _ = make_basic(env, config, make_repo, fp_token=False, fp_remote=True)
+    prod, _ = make_basic(env, config, make_repo, fp_token=False, fp_remote=True)
 
     with prod:
         prod.make_commits(
@@ -110,8 +41,8 @@ def test_no_token(env, config, make_repo):
         "should not have created forward port"
 
 def test_remove_token(env, config, make_repo):
-    proj, prod, _ = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
-    proj.fp_github_token = False
+    prod, _ = make_basic(env, config, make_repo)
+    env['runbot_merge.project'].search([]).fp_github_token = False
 
     with prod:
         prod.make_commits(
@@ -132,7 +63,7 @@ def test_remove_token(env, config, make_repo):
         "should not have created forward port"
 
 def test_no_target(env, config, make_repo):
-    proj, prod, _ = make_basic(env, config, make_repo, fp_token=True, fp_remote=False)
+    prod, _ = make_basic(env, config, make_repo, fp_remote=False)
 
     with prod:
         prod.make_commits(
@@ -153,7 +84,7 @@ def test_no_target(env, config, make_repo):
         "should not have created forward port"
 
 def test_failed_staging(env, config, make_repo):
-    proj, prod, _ = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
+    prod, _ = make_basic(env, config, make_repo)
 
     reviewer = config['role_reviewer']['token']
     with prod:
@@ -421,8 +352,9 @@ def test_new_intermediate_branch(env, config, make_repo):
     def validate(repo, commit):
         repo.post_status(commit, 'success', 'ci/runbot')
         repo.post_status(commit, 'success', 'legal/cla')
-    project, prod, _ = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
-    _, prod2, _ = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
+    prod, _ = make_basic(env, config, make_repo)
+    prod2, _ = make_basic(env, config, make_repo)
+    project = env['runbot_merge.project'].search([])
     assert len(project.repo_ids) == 2
 
     original_c_tree = prod.read_tree(prod.commit('c'))
@@ -599,7 +531,7 @@ def test_new_intermediate_branch(env, config, make_repo):
     }, "check that new got all the updates (should be in the same state as c really)"
 
 def test_author_can_close_via_fwbot(env, config, make_repo):
-    project, prod, xxx = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
+    prod, _ = make_basic(env, config, make_repo)
     other_user = config['role_other']
     other_token = other_user['token']
     other = prod.fork(token=other_token)
@@ -642,7 +574,7 @@ def test_author_can_close_via_fwbot(env, config, make_repo):
     assert pr1_id.state == 'closed'
 
 def test_skip_ci_all(env, config, make_repo):
-    project, prod, _ = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
+    prod, _ = make_basic(env, config, make_repo)
 
     with prod:
         prod.make_commits('a', Commit('x', tree={'x': '0'}), ref='heads/change')
@@ -674,7 +606,7 @@ def test_skip_ci_all(env, config, make_repo):
     assert pr2_id.source_id == pr0_id
 
 def test_skip_ci_next(env, config, make_repo):
-    project, prod, _ = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
+    prod, _ = make_basic(env, config, make_repo)
 
     with prod:
         prod.make_commits('a', Commit('x', tree={'x': '0'}), ref='heads/change')
@@ -712,7 +644,8 @@ def test_retarget_after_freeze(env, config, make_repo, users):
     latter port. In that case the reinsertion task should just do nothing, and
     the retargeted PR should be forward-ported normally once merged.
     """
-    project, prod, _ = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
+    prod, _ = make_basic(env, config, make_repo)
+    project = env['runbot_merge.project'].search([])
     with prod:
         [c] = prod.make_commits('b', Commit('thing', tree={'x': '1'}), ref='heads/mypr')
         pr = prod.make_pr(target='b', head='mypr')
@@ -788,7 +721,7 @@ def test_retarget_after_freeze(env, config, make_repo, users):
     assert new_pr_id.target == branch_c
 
 def test_approve_draft(env, config, make_repo, users):
-    _, prod, _ = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
+    prod, _ = make_basic(env, config, make_repo)
 
     with prod:
         prod.make_commits('a', Commit('x', tree={'x': '0'}), ref='heads/change')
@@ -817,7 +750,8 @@ def test_freeze(env, config, make_repo, users):
 
     - should not forward-port the freeze PRs themselves
     """
-    project, prod, _ = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
+    prod, _ = make_basic(env, config, make_repo)
+    project = env['runbot_merge.project'].search([])
     # branches here are "a" (older), "b", and "c" (master)
     with prod:
         [root, _] = prod.make_commits(
@@ -871,7 +805,7 @@ def test_missing_magic_ref(env, config, make_repo):
     Emulate this behaviour by updating the PR with a commit which lives in the
     repo but has no ref.
     """
-    _, prod, _ = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
+    prod, _ = make_basic(env, config, make_repo)
     a_head = prod.commit('refs/heads/a')
     with prod:
         [c] = prod.make_commits(a_head.id, Commit('x', tree={'x': '0'}), ref='heads/change')
@@ -925,8 +859,8 @@ def test_disable_branch_with_batches(env, config, make_repo, users):
     been forward ported yet port them over, as if their source had been merged
     after the branch was disabled (thus skipped over)
     """
-    proj, repo, fork = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
-    proj.repo_ids.required_statuses = "default"
+    repo, fork = make_basic(env, config, make_repo, statuses="default")
+    proj = env['runbot_merge.project'].search([])
     branch_b = env['runbot_merge.branch'].search([('name', '=', 'b')])
     assert branch_b
 
@@ -1123,8 +1057,7 @@ def test_maintain_batch_history(env, config, make_repo, users):
     batch as each PR being closed (until the last one?) will be removed from
     the batch.
     """
-    proj, repo, fork = make_basic(env, config, make_repo, fp_token=True, fp_remote=True)
-    proj.repo_ids.required_statuses = "default"
+    repo, fork = make_basic(env, config, make_repo, statuses="default")
 
     with repo, fork:
         fork.make_commits("a", Commit("x", tree={"x": "1"}), ref="heads/x")
diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index 5da2a744..f72b286a 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -55,16 +55,51 @@ class re_matches:
 def seen(env, pr, users):
     return users['user'], f'[Pull request status dashboard]({to_pr(env, pr).url}).'
 
-def make_basic(env, config, make_repo, *, reponame='proj', project_name='myproject'):
-    """ Creates a basic repo with 3 forking branches
+def make_basic(
+        env,
+        config,
+        make_repo,
+        *,
+        project_name='myproject',
+        reponame='proj',
+        statuses='legal/cla,ci/runbot',
+        fp_token=True,
+        fp_remote=True,
+):
+    """ Creates a project ``project_name`` **if none exists**, otherwise
+    retrieves the existing one and adds a new repository and its fork.
+
+    Repositories are setup with three forking branches:
+
+    ::
+
+        f = 0 -- 1 -- 2 -- 3 -- 4  : a
+                      |
+        g =           `-- 11 -- 22 : b
+                         |
+        h =               `-- 111  : c
 
-    f = 0 -- 1 -- 2 -- 3 -- 4  : a
-                  |
-    g =           `-- 11 -- 22 : b
-                     |
-    h =               `-- 111  : c
     each branch just adds and modifies a file (resp. f, g and h) through the
     contents sequence a b c d e
+
+    :param env: Environment, for odoo model interactions
+    :param config: pytest project config thingie
+    :param make_repo: repo maker function, normally the fixture, should be a
+                      ``Callable[[str], Repo]``
+    :param project_name: internal project name, can be used to recover the
+                         project object afterward, matches exactly since it's
+                         unique per odoo db (and thus test)
+    :param reponame: the base name of the repository, for identification, for
+                     concurrency reasons the actual repository name *will* be
+                     different
+    :param statuses: required statuses for the repository, stupidly default to
+                     the old Odoo statuses, should be moved to ``default`` over
+                     time for simplicity (unless the test specifically calls for
+                     multiple statuses)
+    :param fp_token: whether to set the ``fp_github_token`` on the project if
+                     / when creating it
+    :param fp_remote: whether to create a fork repo and set it as the
+                      repository's ``fp_remote_target``
     """
     Projects = env['runbot_merge.project']
     project = Projects.search([('name', '=', project_name)])
@@ -73,7 +108,7 @@ def make_basic(env, config, make_repo, *, reponame='proj', project_name='myproje
             'name': project_name,
             'github_token': config['github']['token'],
             'github_prefix': 'hansen',
-            'fp_github_token': config['github']['token'],
+            'fp_github_token': fp_token and config['github']['token'],
             'fp_github_name': 'herbert',
             'branch_ids': [
                 (0, 0, {'name': 'a', 'sequence': 100}),
@@ -104,12 +139,12 @@ def make_basic(env, config, make_repo, *, reponame='proj', project_name='myproje
             Commit('111', tree={'h': 'a'}),
             ref='heads/c',
         )
-    other = prod.fork()
+    other = prod.fork() if fp_remote else None
     repo = env['runbot_merge.repository'].create({
         'project_id': project.id,
         'name': prod.name,
-        'required_statuses': 'legal/cla,ci/runbot',
-        'fp_remote_target': other.name,
+        'required_statuses': statuses,
+        'fp_remote_target': other.name if other else False,
     })
     env['res.partner'].search([
         ('github_login', '=', config['role_reviewer']['user'])

From 1e9fa48652c0c97d20ef085625f59b8c23239c0c Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 27 Feb 2024 10:07:11 +0100
Subject: [PATCH 098/192] [ADD] runbot_merge: migration of models refactoring

This is definitely non-trivial, due to the structural changes and the
amounts of stuff to move around (e.g. lift from PR to batch), as well
as the reification of previously non-existent relations (batches,
batch history, ...) which sometimes uncovers inconsistencies in the
current state of the mergebot (some of which are the result of bugs,
the bug got fixed but the nonsense it generated was left untouched).
---
 runbot_merge/__manifest__.py                  |   2 +-
 .../migrations/15.0.1.12/pre-migration.py     | 728 ++++++++++++++++++
 2 files changed, 729 insertions(+), 1 deletion(-)
 create mode 100644 runbot_merge/migrations/15.0.1.12/pre-migration.py

diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py
index bcb702b6..9799907d 100644
--- a/runbot_merge/__manifest__.py
+++ b/runbot_merge/__manifest__.py
@@ -1,6 +1,6 @@
 {
     'name': 'merge bot',
-    'version': '1.11',
+    'version': '1.12',
     'depends': ['contacts', 'mail', 'website'],
     'data': [
         'security/security.xml',
diff --git a/runbot_merge/migrations/15.0.1.12/pre-migration.py b/runbot_merge/migrations/15.0.1.12/pre-migration.py
new file mode 100644
index 00000000..12762afe
--- /dev/null
+++ b/runbot_merge/migrations/15.0.1.12/pre-migration.py
@@ -0,0 +1,728 @@
+"""This is definitely the giantest of fucks as pretty much the entire model was
+reworked
+"""
+import dataclasses
+import logging
+from collections import defaultdict
+from itertools import chain
+from typing import TypeVar, Any
+
+from psycopg2.extras import execute_batch, execute_values
+from psycopg2.sql import SQL
+
+logger = logging.getLogger("odoo.modules.migration.runbot_merge.15.0.1.12")
+
+def cleanup(cr):
+    """There seems to be some *pretty* weird database state having crept
+    """
+    # Until 2021 (not sure why exactly) a bunch of batches were created with no
+    # PRs, some staged and some not.
+    logger.info("Delete batches without PRs...")
+    cr.execute("""
+    DELETE FROM runbot_merge_batch
+    WHERE id IN (
+        SELECT b.id
+        FROM runbot_merge_batch b
+        LEFT JOIN runbot_merge_batch_runbot_merge_pull_requests_rel r ON (b.id = r.runbot_merge_batch_id)
+        WHERE r.runbot_merge_batch_id IS NULL
+    )
+    """)
+    # some of the batches above were the only ones of their stagings
+    logger.info("Delete stagings without batches...")
+    cr.execute("""
+    DELETE FROM runbot_merge_stagings
+    WHERE id IN (
+        SELECT s.id
+        FROM runbot_merge_stagings s
+        LEFT JOIN runbot_merge_batch b ON (s.id = b.staging_id)
+        WHERE b.id IS NULL
+    )
+    """)
+
+    # check PRs whose source has a source
+    cr.execute("""
+    SELECT
+        p.id AS id,
+        s.id AS source_id,
+        r.name || '#' || p.number AS pr,
+        pr.name || '#' || pp.number AS parent,
+        sr.name || '#' || s.number AS source
+
+    FROM runbot_merge_pull_requests p
+    JOIN runbot_merge_repository r ON (r.id = p.repository)
+
+    JOIN runbot_merge_pull_requests pp ON (pp.id = p.source_id)
+    JOIN runbot_merge_repository pr ON (pr.id = pp.repository)
+
+    JOIN runbot_merge_pull_requests s ON (s.id = pp.source_id)
+    JOIN runbot_merge_repository sr ON (sr.id = s.repository)
+    ORDER BY p.id;
+    """)
+    for pid, ssid, _, _, _ in cr.fetchall():
+        cr.execute("UPDATE runbot_merge_pull_requests SET source_id = %s WHERE id = %s", [ssid, pid])
+
+def hlink(url):
+    """A terminal hlink starts with OSC8;{params};{link}ST and ends with the
+    sequence with no params or link
+    """
+    return f'\x9d8;;{url}\x9c'
+
+def link(label, url):
+    return f"{hlink(url)}{label}{hlink('')}"
+
+def migrate(cr, version):
+    cr.execute("select 1 from forwardport_batches")
+    assert not cr.rowcount, f"can't migrate the mergebot with enqueued forward ports (found {cr.rowcount})"
+    # avoid SQL taking absolutely ungodly amounts of time
+    cr.execute("SET statement_timeout = '60s'")
+    # will be recreated & computed on the fly
+    cr.execute("ALTER TABLE runbot_merge_batch DROP COLUMN target")
+
+    cleanup(cr)
+
+    cr.execute("""
+    SELECT
+        source_name,
+        array_agg(json_build_array(gs.target, gs.prs) order by gs.seq desc)
+    FROM (
+        SELECT
+            rr.name || '#' || source.number as source_name,
+            t.sequence as seq,
+            t.name as target,
+            array_agg(json_build_array(r.name || '#' || p.number, p.state)) as prs
+
+        FROM runbot_merge_pull_requests p
+        JOIN runbot_merge_repository r ON (r.id = p.repository)
+        JOIN runbot_merge_branch t ON (t.id = p.target)
+
+        JOIN runbot_merge_pull_requests source ON (source.id = p.source_id)
+        JOIN runbot_merge_repository rr ON (rr.id = source.repository)
+
+        GROUP BY source.id, rr.id, t.id
+        HAVING count(*) FILTER (WHERE p.state = 'merged') > 1
+    ) gs
+    GROUP BY source_name
+    """)
+    if cr.rowcount:
+        msg = "Found inconsistent batches, which will confuse later chaining\n\n"
+        for source, per_target in cr._obj:
+            msg += f"source {source}\n"
+            for target, prs in per_target:
+                msg += "\t{} {}\n".format(
+                    target,
+                    ", ".join(f'{p} ({s})' for p, s in prs),
+                )
+        raise Exception(msg)
+
+    logger.info("add batch columns...")
+    cr.execute("""
+    CREATE TYPE runbot_merge_batch_priority
+        AS ENUM ('default', 'priority', 'alone');
+
+    ALTER TABLE runbot_merge_batch
+        -- backfilled from staging
+        ADD COLUMN merge_date timestamp,
+        -- backfilled from PRs
+        ADD COLUMN priority runbot_merge_batch_priority NOT NULL DEFAULT 'default',
+        ADD COLUMN skipchecks boolean NOT NULL DEFAULT false,
+        ADD COLUMN cancel_staging boolean NOT NULL DEFAULT false,
+        ADD COLUMN fw_policy varchar NOT NULL DEFAULT 'default'
+    ;
+    """)
+    # batches not linked to stagings are likely to be useless
+    logger.info("add batch/staging join table...")
+    cr.execute("""
+    CREATE TABLE runbot_merge_staging_batch (
+        id serial PRIMARY KEY,
+        runbot_merge_batch_id integer NOT NULL REFERENCES runbot_merge_batch(id) ON DELETE CASCADE,
+        runbot_merge_stagings_id integer NOT NULL REFERENCES runbot_merge_stagings(id) ON DELETE CASCADE
+    );
+    CREATE UNIQUE INDEX runbot_merge_staging_batch_idx ON runbot_merge_staging_batch
+        (runbot_merge_stagings_id, runbot_merge_batch_id);
+    CREATE INDEX runbot_merge_staging_batch_rev ON runbot_merge_staging_batch
+        (runbot_merge_batch_id) INCLUDE (runbot_merge_stagings_id);
+    """)
+    # old 'bot creates a new batch at staging time, associated with that
+    # specific staging, the way to recoup them (to the best of our ability) is
+    # to assume a new style batch is a set of PRs, so if we group batches by prs
+    # we get more or less the set of relevant batches / stagings
+    logger.info("collect batches...")
+    clusters, to_batch = collate_real_batches(cr)
+
+    logger.info("collate batches...")
+    to_delete = []
+    batch_staging_links = []
+    to_rejoin = []
+    for cluster in clusters.clusters:
+        first = cluster.merged_batch or min(cluster.batches)
+        to_delete.extend(cluster.batches - {first})
+        # link all the PRs back to that batch
+        to_rejoin.append((first, list(cluster.prs)))
+        # link `first` to `staging`, ordering insertions by `batch` in order
+        # to conserve batching order
+        batch_staging_links.extend(
+            (batch, first, staging)
+            for batch, staging in cluster.stagings
+        )
+
+    logger.info("link batches to stagings...")
+    # sort (unique_batch, staging) by initial batch so that we create the new
+    # bits in the correct order hopefully
+    batch_staging_links.sort()
+    execute_values(
+        cr._obj,
+        "INSERT INTO runbot_merge_staging_batch (runbot_merge_batch_id, runbot_merge_stagings_id) VALUES %s",
+        ((b, s) for _, b, s in batch_staging_links),
+        page_size=1000,
+    )
+
+    logger.info("detach PRs from \"active\" batches...")
+    # there are non-deactivated batches floating around, which are not linked
+    # to stagings, they seem linked to updates (forward-ported PRs getting
+    # updated), but not exclusively
+    cr.execute("UPDATE runbot_merge_pull_requests SET batch_id = NULL WHERE batch_id IS NOT NULL")
+    # drop constraint because pg checks it even though we've set all the active batches to null
+    cr.execute("ALTER TABLE runbot_merge_pull_requests DROP CONSTRAINT runbot_merge_pull_requests_batch_id_fkey")
+
+    while to_delete:
+        ds, to_delete = to_delete[:10000], to_delete[10000:]
+        logger.info("delete %d leftover batches", len(ds))
+        cr.execute("DELETE FROM runbot_merge_batch WHERE id = any(%s)", [ds])
+
+    logger.info("delete staging column...")
+    cr.execute("ALTER TABLE runbot_merge_batch DROP COLUMN staging_id;")
+
+    logger.info("relink PRs...")
+    cr.execute("DROP TABLE runbot_merge_batch_runbot_merge_pull_requests_rel")
+    execute_batch(
+        cr._obj,
+        "UPDATE runbot_merge_pull_requests SET batch_id = %s WHERE id = any(%s)",
+        to_rejoin,
+        page_size=1000,
+    )
+
+    # at this point all the surviving batches should have associated PRs
+    cr.execute("""
+       SELECT b.id
+         FROM runbot_merge_batch b
+    LEFT JOIN runbot_merge_pull_requests p ON p.batch_id = b.id
+        WHERE p IS NULL;
+    """)
+    if cr.rowcount:
+        logger.error(
+            "All batches should have at least one PR, found %d without",
+            cr.rowcount,
+        )
+
+    # FIXME: fixup PRs marked as merged which don't actually have a batch / staging?
+
+    # the relinked batches are those from stagings, but that means merged PRs
+    # (or at least PRs we tried to merge), we also need batches for non-closed
+    # non-merged PRs
+    logger.info("collect unbatched PRs PRs...")
+    cr.execute("""
+    SELECT
+        CASE
+            -- FIXME: should closed PRs w/o a batch be split out or matched with
+            --        one another?
+            WHEN label SIMILAR TO '%%:patch-[[:digit:]]+'
+                THEN id::text
+            ELSE label
+        END as label_but_not,
+        array_agg(id),
+        array_agg(distinct target)
+    FROM runbot_merge_pull_requests
+    WHERE batch_id IS NULL AND id != all(%s)
+    GROUP BY label_but_not
+    """, [[pid for b in to_batch for pid in b]])
+    for _label, ids, targets in cr._obj:
+        # a few batches are nonsensical e.g. multiple PRs on different
+        # targets from th same branch or mix of master upgrade and stable
+        # branch community, split them out
+        if len(targets) > 1:
+            to_batch.extend([id] for id in ids)
+        else:
+            to_batch.append(ids)
+
+    logger.info("create %d new batches for unbatched prs...", len(to_batch))
+    cr.execute(
+        SQL("INSERT INTO runbot_merge_batch VALUES {} RETURNING id").format(
+            SQL(", ").join([SQL("(DEFAULT)")]*len(to_batch))))
+    logger.info("link unbatched PRs to batches...")
+    execute_batch(
+        cr._obj,
+        "UPDATE runbot_merge_pull_requests SET batch_id = %s WHERE id = any(%s)",
+        [(batch_id, ids) for ids, [batch_id] in zip(to_batch, cr.fetchall())],
+        page_size=1000,
+    )
+
+    cr.execute("SELECT state, count(*) FROM runbot_merge_pull_requests WHERE batch_id IS NULL GROUP BY state")
+    if cr.rowcount:
+        prs = cr.fetchall()
+        logger.error(
+            "Found %d PRs without a batch:%s",
+            sum(c for _, c in prs),
+            "".join(
+                f"\n\t- {c} {p!r} PRs"
+                for p, c in prs
+            ),
+        )
+
+    # FIXME: leverage WEIRD_SEQUENCES
+
+    logger.info("move pr data to batches...")
+    cr.execute("""
+    UPDATE runbot_merge_batch b
+    SET merge_date = v.merge_date,
+        priority = v.p::varchar::runbot_merge_batch_priority,
+        skipchecks = v.skipchecks,
+        cancel_staging = v.cancel_staging,
+        fw_policy = case when v.skipci
+            THEN 'skipci'
+            ELSE 'default'
+        END
+    FROM (
+        SELECT
+            batch_id as id,
+            max(priority) as p,
+            min(merge_date) as merge_date,
+            -- added to PRs in 1.11 so can be aggregated & copied over
+            bool_or(skipchecks) as skipchecks,
+            bool_or(cancel_staging) as cancel_staging,
+            bool_or(fw_policy = 'skipci') as skipci
+        FROM runbot_merge_pull_requests
+        GROUP BY batch_id
+    ) v
+    WHERE b.id = v.id
+    """)
+
+    logger.info("restore batch constraint...")
+    cr.execute("""
+    ALTER TABLE runbot_merge_pull_requests
+        ADD CONSTRAINT runbot_merge_pull_requests_batch_id_fkey
+        FOREIGN KEY (batch_id)
+        REFERENCES runbot_merge_batch (id)
+    """)
+
+    # remove xid for x_prs (not sure why it exists)
+    cr.execute("""
+    DELETE FROM ir_model_data
+       WHERE module = 'forwardport'
+         AND name = 'field_forwardport_batches__x_prs'
+    """)
+    # update (x_)prs to match the updated field type(s)
+    cr.execute("""
+    UPDATE ir_model_fields
+       SET ttype = 'one2many',
+           relation = 'runbot_merge.pull_requests',
+           relation_field = 'batch_id'
+     WHERE model_id = 445 AND name = 'prs';
+
+    UPDATE ir_model_fields
+       SET ttype = 'one2many'
+     WHERE model_id = 448 AND name = 'x_prs';
+    """)
+
+    logger.info("generate batch parenting...")
+    cr.execute("SELECT id, project_id, name FROM runbot_merge_branch ORDER BY project_id, sequence, name")
+    # branch_id -> str
+    branch_names = {}
+    # branch_id -> project_id
+    projects = {}
+    # project_id -> list[branch_id]
+    branches_for_project = {}
+    for bid, pid, name in cr._obj:
+        branch_names[bid] = name
+        projects[bid] = pid
+        branches_for_project.setdefault(pid, []).append(bid)
+    cr.execute("""
+    SELECT batch_id,
+           array_agg(distinct target),
+           array_agg(json_build_object(
+               'id', p.id,
+               'name', r.name || '#' || number,
+               'repo', r.name,
+               'number', number,
+               'state', p.state,
+               'source', source_id
+           ))
+    FROM runbot_merge_pull_requests p
+    JOIN runbot_merge_repository r ON (r.id = p.repository)
+    GROUP BY batch_id
+    """)
+    todos = []
+    descendants = defaultdict(list)
+    targets = {}
+    batches = {}
+    batch_prs = {}
+    for batch, target_ids, prs in cr._obj:
+        assert len(target_ids) == 1, \
+            "Found batch with multiple targets {tnames} {prs}".format(
+                tnames=', '.join(branch_names[id] for id in target_ids),
+                prs=prs,
+            )
+
+        todos.append((batch, target_ids[0], prs))
+        batch_prs[batch] = prs
+        for pr in prs:
+            pr['link'] = link(pr['name'], "https://mergebot.odoo.com/{repo}/pull/{number}".format_map(pr))
+
+            targets[pr['id']] = target_ids[0]
+            batches[pr['id']] = batch
+            batches[pr['name']] = batch
+            if pr['source']:
+                descendants[pr['source']].append(pr['id'])
+            else:
+                # put source PRs as their own descendants otherwise the linkage
+                # fails when trying to find the top-most parent
+                descendants[pr['id']].append(pr['id'])
+    assert None not in descendants
+
+    for prs in chain(
+        KNOWN_BATCHES,
+        chain.from_iterable(WEIRD_SEQUENCES),
+    ):
+        batch_of_prs = {batches[f'odoo/{p}'] for p in prs}
+        assert len(batch_of_prs) == 1,\
+            "assumed {prs} were the same batch, got {batch_of_prs}".format(
+                prs=', '.join(prs),
+                batch_of_prs='; '.join(
+                    '{} => {}'.format(p, batches[f'odoo/{p}'])
+                    for p in prs
+                )
+            )
+
+        prs_of_batch = {pr['name'].removeprefix('odoo/') for pr in batch_prs[batch_of_prs.pop()]}
+        assert set(prs) == prs_of_batch,\
+            "assumed batch would contain {prs}, got {prs_of_batch}".format(
+                prs=', '.join(prs),
+                prs_of_batch=', '.join(prs_of_batch),
+            )
+
+    parenting = []
+    for batch, target, prs in todos:
+        sources = [p['source'] for p in prs if p['source']]
+        # can't have parent batch without source PRs
+        if not sources:
+            continue
+
+        pid = projects[target]
+        branches = branches_for_project[pid]
+
+        # we need all the preceding targets in order to jump over disabled branches
+        previous_targets = branches[branches.index(target) + 1:]
+        if not previous_targets:
+            continue
+
+        for previous_target in previous_targets:
+            # from each source, find the descendant targeting the earlier target,
+            # then get the batch of these PRs
+            parents = {
+                batches[descendant]
+                for source in sources
+                for descendant in descendants[source]
+                if targets[descendant] == previous_target
+            }
+            if parents:
+                break
+        else:
+            continue
+
+        if len(parents) == 2:
+            parents1, parents2 = [batch_prs[parent] for parent in parents]
+            # if all of one parent are merged and all of the other are not, take the merged side
+            if all(p['state'] == 'merged' for p in parents1) and all(p['state'] != 'merged' for p in parents2):
+                parents = [list(parents)[0]]
+            elif all(p['state'] != 'merged' for p in parents1) and all(p['state'] == 'merged' for p in parents2):
+                parents = [list(parents)[1]]
+            elif len(parents1) == 1 and len(parents2) == 1 and len(prs) == 1:
+                # if one of the candidates is older than the current PR
+                # (lower id) and the other one younger, assume the first one is
+                # correct
+                p = min(parents, key=lambda p: batch_prs[p][0]['id'])
+                low = batch_prs[p]
+                high = batch_prs[max(parents, key=lambda p: batch_prs[p][0]['id'])]
+                if low[0]['id'] < prs[0]['id'] < high[0]['id']:
+                    parents = [p]
+
+        if real_parents := SAAS_135_INSERTION_CONFUSION.get(tuple(sorted(parents))):
+            parents = real_parents
+
+        assert len(parents) == 1,\
+            ("Found multiple candidates for batch {batch} ({prs})"
+             " with target {target} (previous={previous_target})\n\t{parents}".format(
+                parents="\n\t".join(
+                    "{} ({})".format(
+                        parent,
+                        ", ".join(
+                            f"{p['link']} ({p['state']}, {branch_names[targets[p['id']]]})"
+                            for p in batch_prs[parent]
+                        )
+                    )
+                    for parent in parents
+                ),
+                batch=batch,
+                target=branch_names[target],
+                previous_target=branch_names[previous_target],
+                prs=', '.join(map("{link} ({state})".format_map, prs)),
+            ))
+        parenting.append((parents.pop(), batch))
+
+    logger.info("set batch parenting...")
+    # add column down here otherwise the FK constraint has to be verified for
+    # each batch we try to delete and that is horrendously slow, deferring the
+    # constraints is not awesome because we need to check it at the first DDL
+    # and that's still way slower than feels necessary
+    cr.execute("""
+    ALTER TABLE runbot_merge_batch
+        ADD COLUMN parent_id integer
+        REFERENCES runbot_merge_batch(id)
+    """)
+    execute_batch(
+        cr._obj,
+        "UPDATE runbot_merge_batch SET parent_id = %s WHERE id = %s",
+        parenting,
+        page_size=1000,
+    )
+
+@dataclasses.dataclass(slots=True, kw_only=True)
+class Cluster:
+    merged_batch: int | None = None
+    prs: set[int] = dataclasses.field(default_factory=set)
+    batches: set[int] = dataclasses.field(default_factory=set)
+    stagings: set[tuple[int, int]] = dataclasses.field(default_factory=set)
+    "set of original (batch, staging) pairs"
+
+@dataclasses.dataclass
+class Clusters:
+    clusters: list[Cluster] = dataclasses.field(default_factory=list)
+    by_batch: dict[int, Cluster] = dataclasses.field(default_factory=dict)
+    by_pr: dict[int, Cluster] = dataclasses.field(default_factory=dict)
+
+@dataclasses.dataclass(slots=True, kw_only=True)
+class Batch:
+    staging: int | None = None
+    merged: bool = False
+    prs: set[int] = dataclasses.field(default_factory=set)
+
+T = TypeVar('T')
+def insert(s: set[T], v: T) -> bool:
+    """Inserts v in s if not in, and returns whether an insertion was needed.
+    """
+    if v in s:
+        return False
+    else:
+        s.add(v)
+        return True
+def collate_real_batches(cr: Any) -> tuple[Clusters, list[list[int]]]:
+    cr.execute('''
+    SELECT
+        st.id as staging,
+        st.state as staging_state,
+        b.id as batch_id,
+        p.id as pr_id
+    FROM runbot_merge_batch_runbot_merge_pull_requests_rel br
+    JOIN runbot_merge_batch b ON (b.id = br.runbot_merge_batch_id)
+    JOIN runbot_merge_pull_requests as p ON (p.id = br.runbot_merge_pull_requests_id)
+    LEFT JOIN runbot_merge_stagings st ON (st.id = b.staging_id)
+    ''')
+    batch_map: dict[int, Batch] = {}
+    pr_to_batches = defaultdict(set)
+    for staging_id, staging_state, batch_id, pr_id in cr.fetchall():
+        pr_to_batches[pr_id].add(batch_id)
+
+        if batch := batch_map.get(batch_id):
+            batch.prs.add(pr_id)
+        else:
+            batch_map[batch_id] = Batch(
+                staging=staging_id,
+                merged=staging_state == 'success',
+                prs={pr_id},
+            )
+
+    # maps a PR name to its id
+    cr.execute("""
+    SELECT r.name || '#' || p.number, p.id
+      FROM runbot_merge_pull_requests p
+      JOIN runbot_merge_repository r ON (r.id = p.repository)
+     WHERE r.name || '#' || p.number = any(%s)
+    """, [[f'odoo/{p}' for seq in WEIRD_SEQUENCES for b in seq if len(b) > 1 for p in b]])
+    prmap: dict[str, int] = dict(cr._obj)
+    to_batch = []
+    # for each WEIRD_SEQUENCES batch, we need to merge their batches if any,
+    # and create them otherwise
+    for batch in (b for seq in WEIRD_SEQUENCES for b in seq if len(b) > 1):
+        ids = [prmap[f'odoo/{n}'] for n in batch]
+        batches = {b for pid in ids for b in pr_to_batches[pid]}
+        if batches:
+            for pid in ids:
+                pr_to_batches[pid].update(batches)
+            for bid in batches:
+                batch_map[bid].prs.update(ids)
+        else:
+            # need to create a new batch
+            to_batch.append(ids)
+
+    clusters = Clusters()
+    # we can start from either the PR or the batch side to reconstruct a cluster
+    for pr_id in pr_to_batches:
+        if pr_id in clusters.by_pr:
+            continue
+
+        to_visit = [pr_id]
+        prs: set[int] = set()
+        merged_batch = None
+        batches: set[int] = set()
+        stagings: set[tuple[int, int]] = set()
+        while to_visit:
+            pr_id = to_visit.pop()
+            if not insert(prs, pr_id):
+                continue
+
+            for batch_id in pr_to_batches[pr_id]:
+                if not insert(batches, batch_id):
+                    continue
+
+                b = batch_map[batch_id]
+                if s := b.staging:
+                    stagings.add((batch_id, s))
+                if b.merged:
+                    merged_batch = batch_id
+                to_visit.extend(b.prs - prs)
+
+        c = Cluster(merged_batch=merged_batch, prs=prs, batches=batches, stagings=stagings)
+        clusters.clusters.append(c)
+        clusters.by_batch.update((batch_id, c) for batch_id in c.batches)
+        clusters.by_pr.update((pr_id, c) for pr_id in c.prs)
+
+    return clusters, to_batch
+
+# at the creation of saas 13.5, the forwardbot clearly got very confused and
+# somehow did not correctly link the PRs it reinserted together, leading to
+# some of them being merged separately, leading the batch parenting linker thing
+# to be extremely confused
+SAAS_135_INSERTION_CONFUSION = {
+    (48200, 48237): [48237],
+    (48353, 48388): [48353],
+    (48571, 48602): [48602],
+    (73614, 73841): [73614],
+}
+
+KNOWN_BATCHES = [
+    # both closed, same source (should be trivial)
+    ["odoo#151827", "enterprise#55453"],
+    ["odoo#66743", "enterprise#16631"],
+
+    # both closed but different sources
+    ["odoo#57659", "enterprise#13204"],
+    ["odoo#57752", "enterprise#13238"],
+    ["odoo#94152", "enterprise#28664"],
+    ["odoo#114059", "enterprise#37690"],
+    ["odoo#152904", "enterprise#55975"],
+
+    # one closed the other not, different sources (so a PR was added in the
+    # middle of a forward port then its descendant was closed evn though the
+    # other repo / sequence kept on keeping)
+    ["odoo#113422", "enterprise#37429"],
+    ["odoo#151992", "enterprise#55501"],
+    ["odoo#159211", "enterprise#59407"],
+
+    # closed without a sibling but their source had a sibling
+    ["odoo#67727"], # enterprise closed at enterprise#16631
+    ["odoo#70828"], # enterprise closed at enterprise#17901
+    ["odoo#132817"], # enterprise closed at enterprise#44656
+    ["odoo#137855"], # enterprise closed at enterprise#48092
+    ["enterprise#49430"], # odoo closed at odoo#139515
+
+    ["odoo#109811", "enterprise#35966"],
+    ["odoo#110311", "enterprise#35983"],
+    ["odoo#110576"],
+]
+
+# This is next level weird compared to the previous so it gets extra care:
+# these are sequences with multiple points of divergence or grafting
+WEIRD_SEQUENCES = [
+    [
+        ["odoo#40466"],
+        ["odoo#40607"],
+        ["odoo#40613", "odoo#41106"],
+        ["odoo#40615", "odoo#41112"],
+        ["odoo#40627", "odoo#41116", "odoo#41163"],
+        ["odoo#40638", "odoo#41119", "odoo#41165"],
+    ],
+    [
+        ["odoo#46405"],
+        ["odoo#46698"],
+        ["odoo#46820"],
+        ["odoo#46974"],
+        ["odoo#47273"],
+        ["odoo#47345",               "enterprise#9259"],
+        ["odoo#47349", "odoo#47724", "enterprise#9274"],
+    ],
+    [
+        ["odoo#47923"],
+        ["odoo#47986"],
+        ["odoo#47991", "odoo#48010"],
+        ["odoo#47996", "odoo#48015", "odoo#48016"],
+        ["odoo#48003"],
+    ],
+    [
+        ["enterprise#9996"],
+        ["enterprise#10062", "odoo#49828"],
+        ["enterprise#10065", "odoo#49852", "enterprise#10076"],
+        ["enterprise#10173", "odoo#50087"],
+        ["enterprise#10179", "odoo#50104"],
+        ["enterprise#10181", "odoo#50110"],
+    ],
+    [
+        ["enterprise#16357"],
+        ["enterprise#16371"],
+        ["enterprise#16375", "enterprise#16381"],
+        ["enterprise#16378", "enterprise#16385"],
+        ["enterprise#16379", "enterprise#16390"],
+    ],
+    [
+        ["odoo#55112"],
+        ["odoo#55120"],
+        ["odoo#55123", "odoo#55159"],
+        ["odoo#55128", "odoo#55169"],
+        ["odoo#55135", "odoo#55171"],
+        ["odoo#55140", "odoo#55172"],
+    ],
+    [
+        ["odoo#56254", "enterprise#12558"],
+        ["odoo#56294", "enterprise#12564"],
+        ["odoo#56300", "enterprise#12566"],
+        ["odoo#56340", "enterprise#12589", "enterprise#12604"],
+        ["odoo#56391",                     "enterprise#12608"],
+    ],
+    [
+        ["enterprise#12565", "odoo#56299"],
+        ["enterprise#12572", "odoo#56309", "odoo#56494"],
+        ["enterprise#12660",               "odoo#56518"],
+        ["enterprise#12688",               "odoo#56581"],
+        ["enterprise#12691"],
+    ],
+    [
+        ["odoo#64706"],
+        ["odoo#65275"],
+        ["odoo#65279", "odoo#65405"],
+        ["odoo#65489", "odoo#65491"],
+    ],
+    [
+        ["odoo#66176"],
+        ["odoo#66188"],
+        ["odoo#66191"],
+        ["odoo#66194", "odoo#66226"],
+        ["odoo#66200", "odoo#66229", "odoo#66277"],
+        ["odoo#66204", "odoo#66232", "odoo#66283"],
+        ["odoo#66208", "odoo#66234", "odoo#66285", "odoo#66303"],
+    ],
+    [
+        ["enterprise#22089", "odoo#79348"],
+        ["enterprise#26736", "odoo#90050"],
+        ["enterprise#31822", "odoo#101218", "odoo#106002"],
+        ["enterprise#36014",                "odoo#110369", "odoo#113892"],
+        ["enterprise#37690",                               "odoo#114059"],
+    ],
+]

From e7e81bf375b71fc402888d091bfcbc82fbbc8a09 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 29 Mar 2024 09:16:51 +0100
Subject: [PATCH 099/192] [IMP] *: handle the addition of a new PR to a
 fw-ported batch

Given a batch which has been merged, and been forward-ported, to
multiple branches (because skipci was set or ci passed on the repos
the batch covers).

There might come the need to add a PR for one of the uncovered
repos. This raises the question of what to do with it, since the
forward-ports for the batch already exist it's not going to get
forwardported normally, nor may we want to, possibly?

Options are:

- don't do anything, such additions don't get ported, this is
  incongruous and unexpected as by default PRs are forward-ported, and
  if the batch wasn't an intermediate (but e.g. a conflict) it
  probably would be ported forward
- port on merge, this allows configuring the PR properly (as it might
  need its own limit) but it means further batches may get
  unexpectedly merged (or at least retied) without the additional PR
  even though we likely want it in
- immediately port the additional PR on creation, this makes the limit
  harder or impossible to configure but it makes the *batch sequence*
  more consistent

We ended up selecting the latter, it feels closer to the updates
system, and it creates more consistent batches through the
sequence. It's also technically easier to ad-hoc port a PR through a
bunch of branches than it is to update the "normal" forward-port
process to handle partial fixups.
---
 forwardport/models/forwardport.py    | 127 +++++++++++++++++-
 forwardport/models/project.py        |  15 ++-
 forwardport/tests/test_batches.py    | 190 ++++++++++++++++++++++++++-
 mergebot_test_utils/utils.py         |   5 +-
 runbot_merge/models/batch.py         |  70 +---------
 runbot_merge/models/pull_requests.py |  59 +++++++++
 6 files changed, 392 insertions(+), 74 deletions(-)

diff --git a/forwardport/models/forwardport.py b/forwardport/models/forwardport.py
index 00dac319..75823461 100644
--- a/forwardport/models/forwardport.py
+++ b/forwardport/models/forwardport.py
@@ -1,9 +1,12 @@
 # -*- coding: utf-8 -*-
+import contextlib
 import logging
+import re
 import uuid
 from contextlib import ExitStack
 from datetime import datetime, timedelta
 
+import requests
 import sentry_sdk
 from dateutil import relativedelta
 
@@ -13,6 +16,7 @@ from odoo.addons.runbot_merge.github import GH
 
 # how long a merged PR survives
 MERGE_AGE = relativedelta.relativedelta(weeks=2)
+FOOTER = '\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n'
 
 _logger = logging.getLogger(__name__)
 
@@ -66,9 +70,11 @@ class ForwardPortTasks(models.Model, Queue):
     source = fields.Selection([
         ('merge', 'Merge'),
         ('fp', 'Forward Port Followup'),
-        ('insert', 'New branch port')
+        ('insert', 'New branch port'),
+        ('complete', 'Complete ported batches'),
     ], required=True)
     retry_after = fields.Datetime(required=True, default='1900-01-01 01:01:01')
+    pr_id = fields.Many2one('runbot_merge.pull_requests')
 
     def _search_domain(self):
         return super()._search_domain() + [
@@ -82,8 +88,11 @@ class ForwardPortTasks(models.Model, Queue):
     def _process_item(self):
         batch = self.batch_id
         sentry_sdk.set_tag('forward-porting', batch.prs.mapped('display_name'))
-        newbatch = batch._port_forward()
+        if self.source == 'complete':
+            self._complete_batches()
+            return
 
+        newbatch = batch._port_forward()
         if not newbatch:  # reached end of seq (or batch is empty)
             # FIXME: or configuration is fucky so doesn't want to FP (maybe should error and retry?)
             _logger.info(
@@ -114,6 +123,120 @@ class ForwardPortTasks(models.Model, Queue):
                 if newchild:
                     newchild.parent_id = pr.id
 
+    def _complete_batches(self):
+        source = pr = self.pr_id
+        if not pr:
+            _logger.warning(
+                "Unable to complete descendants of %s (%s): no new PR",
+                self.batch_id,
+                self.batch_id.prs.mapped('display_name'),
+            )
+            return
+        _logger.info(
+            "Completing batches for descendants of %s (added %s)",
+            self.batch_id.prs.mapped('display_name'),
+            self.pr_id.display_name,
+        )
+
+        gh = requests.Session()
+        repository = pr.repository
+        gh.headers['Authorization'] = f'token {repository.project_id.fp_github_token}'
+        PullRequests = self.env['runbot_merge.pull_requests']
+        self.env.cr.execute('LOCK runbot_merge_pull_requests IN SHARE MODE')
+
+        # TODO: extract complete list of targets from `_find_next_target`
+        #       so we can create all the forwardport branches, push them, and
+        #       only then create the PR objects
+        # TODO: maybe do that after making forward-port WC-less, so all the
+        #       branches can be pushed atomically at once
+        with contextlib.ExitStack() as s:
+            for descendant in self.batch_id.descendants():
+                target = pr._find_next_target()
+                if target is None:
+                    _logger.info("Will not forward-port %s: no next target", pr.display_name)
+                    return
+
+                if PullRequests.search_count([
+                    ('source_id', '=', source.id),
+                    ('target', '=', target.id),
+                    ('state', 'not in', ('closed', 'merged')),
+                ]):
+                    _logger.warning("Will not forward-port %s: already ported", pr.display_name)
+                    return
+
+                if target != descendant.target:
+                    self.env['runbot_merge.pull_requests.feedback'].create({
+                        'repository': repository.id,
+                        'pull_request': source.id,
+                        'token_field': 'fp_github_token',
+                        'message': """\
+{pr.ping}unable to port this PR forwards due to inconsistency: goes from \
+{pr.target.name} to {next_target.name} but {batch} ({batch_prs}) targets \
+{batch.target.name}.
+""".format(pr=pr, next_target=target, batch=descendant, batch_prs=', '.join(descendant.mapped('prs.display_name')))
+                    })
+                    return
+
+                ref = descendant.prs[:1].refname
+                # NOTE: ports the new source everywhere instead of porting each
+                #       PR to the next step as it does not *stop* on conflict
+                conflict, working_copy = source._create_fp_branch(target, ref, s)
+                working_copy.push('target', ref)
+
+                remote_target = repository.fp_remote_target
+                owner, _ = remote_target.split('/', 1)
+                message = source.message + f"\n\nForward-Port-Of: {pr.display_name}"
+
+                title, body = re.match(r'(?P<title>[^\n]+)\n*(?P<body>.*)', message, flags=re.DOTALL).groups()
+                r = gh.post(f'https://api.github.com/repos/{pr.repository.name}/pulls', json={
+                    'base': target.name,
+                    'head': f'{owner}:{ref}',
+                    'title': '[FW]' + (' ' if title[0] != '[' else '') + title,
+                    'body': body
+                })
+                if not r.ok:
+                    _logger.warning("Failed to create forward-port PR for %s, deleting branches", pr.display_name)
+                    # delete all the branches this should automatically close the
+                    # PRs if we've created any. Using the API here is probably
+                    # simpler than going through the working copies
+                    d = gh.delete(f'https://api.github.com/repos/{remote_target}/git/refs/heads/{ref}')
+                    if d.ok:
+                        _logger.info("Deleting %s:%s=success", remote_target, ref)
+                    else:
+                        _logger.warning("Deleting %s:%s=%s", remote_target, ref, d.text)
+                    raise RuntimeError(f"Forwardport failure: {pr.display_name} ({r.text})")
+
+                new_pr = PullRequests._from_gh(r.json())
+                _logger.info("Created forward-port PR %s", new_pr)
+                new_pr.write({
+                    'batch_id': descendant.id, # should already be set correctly but...
+                    'merge_method': pr.merge_method,
+                    'source_id': source.id,
+                    # only link to previous PR of sequence if cherrypick passed
+                    # FIXME: apply parenting of siblings? Apply parenting *to* siblings?
+                    'parent_id': pr.id if not conflict else False,
+                    'detach_reason': "{1}\n{2}".format(*conflict).strip() if conflict else None,
+                })
+
+                if conflict:
+                    self.env.ref('runbot_merge.forwardport.failure.conflict')._send(
+                        repository=pr.repository,
+                        pull_request=pr.number,
+                        token_field='fp_github_token',
+                        format_args={'source': source, 'pr': pr, 'new': new_pr, 'footer': FOOTER},
+                    )
+                new_pr._fp_conflict_feedback(pr, {pr: conflict})
+
+                labels = ['forwardport']
+                if conflict:
+                    labels.append('conflict')
+                self.env['runbot_merge.pull_requests.tagging'].create({
+                    'repository': new_pr.repository.id,
+                    'pull_request': new_pr.number,
+                    'tags_add': labels,
+                })
+
+                pr = new_pr
 
 class UpdateQueue(models.Model, Queue):
     _name = 'forwardport.updates'
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 9fb62a6e..43379eb1 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -17,6 +17,7 @@ import json
 import logging
 import operator
 import subprocess
+import sys
 import tempfile
 import typing
 from pathlib import Path
@@ -195,7 +196,19 @@ class PullRequests(models.Model):
 
         if vals.get('parent_id') and 'source_id' not in vals:
             vals['source_id'] = self.browse(vals['parent_id']).root_id.id
-        return super().create(vals)
+        pr = super().create(vals)
+
+        # added a new PR to an already forward-ported batch: port the PR
+        if self.env['runbot_merge.batch'].search_count([
+            ('parent_id', '=', pr.batch_id.id),
+        ]):
+            self.env['forwardport.batches'].create({
+                'batch_id': pr.batch_id.id,
+                'source': 'complete',
+                'pr_id': pr.id,
+            })
+
+        return pr
 
     def write(self, vals):
         # if the PR's head is updated, detach (should split off the FP lines as this is not the original code)
diff --git a/forwardport/tests/test_batches.py b/forwardport/tests/test_batches.py
index 4c72f3da..0ada435c 100644
--- a/forwardport/tests/test_batches.py
+++ b/forwardport/tests/test_batches.py
@@ -1,4 +1,8 @@
-from utils import Commit, make_basic, to_pr
+import re
+
+import pytest
+
+from utils import Commit, make_basic, to_pr, seen, re_matches
 
 
 def test_single_updated(env, config, make_repo):
@@ -133,19 +137,17 @@ def test_closing_during_fp(env, config, make_repo, users):
 def test_add_pr_during_fp(env, config, make_repo, users):
     """ It should be possible to add new PRs to an FP batch
     """
-    r1, _ = make_basic(env, config, make_repo)
-    r2, fork2 = make_basic(env, config, make_repo)
-    repos = env['runbot_merge.repository'].search([])
-    repos.required_statuses = 'default'
+    r1, _ = make_basic(env, config, make_repo, statuses="default")
+    r2, fork2 = make_basic(env, config, make_repo, statuses="default")
     # needs a "d" branch
-    repos[0].project_id.write({
+    env['runbot_merge.project'].search([]).write({
         'branch_ids': [(0, 0, {'name': 'd', 'sequence': 40})],
     })
     with r1, r2:
         r1.make_ref("heads/d", r1.commit("c").id)
         r2.make_ref("heads/d", r2.commit("c").id)
 
-    with r1, r2:
+    with r1:
         r1.make_commits('a', Commit('1', tree={'1': '0'}), ref='heads/aref')
         pr1_a = r1.make_pr(target='a', head='aref')
         r1.post_status('aref', 'success')
@@ -211,3 +213,177 @@ def test_add_pr_during_fp(env, config, make_repo, users):
 
     assert find_child(pr1_c_id)
     assert find_child(pr2_c_id)
+
+def test_add_to_forward_ported(env, config, make_repo, users):
+    """Add a new branch to an intermediate step of a fw *sequence*, either
+    because skipci or because all the intermediate CI succeeded
+    """
+    # region setup
+    r1, _ = make_basic(env, config, make_repo, statuses="default")
+    r2, fork2 = make_basic(env, config, make_repo, statuses="default")
+
+    with r1:
+        r1.make_commits('a', Commit('a', tree={'a': 'a'}), ref="heads/pr1")
+        pr1_a = r1.make_pr(target="a", head="pr1")
+        r1.post_status(pr1_a.head, 'success')
+        pr1_a.post_comment('hansen r+', config['role_reviewer']['token'])
+    env.run_crons()
+    with r1, r2:
+        r1.post_status('staging.a', 'success')
+        r2.post_status('staging.a', 'success')
+    env.run_crons()
+
+    # region port forward
+    pr1_a_id = to_pr(env, pr1_a)
+    pr1_b_id = pr1_a_id.forwardport_ids
+    assert pr1_b_id
+    with r1:
+        r1.post_status(pr1_b_id.head, 'success')
+    env.run_crons()
+    pr1_c_id = pr1_a_id.forwardport_ids - pr1_b_id
+    assert pr1_c_id
+    # endregion
+    # endregion
+
+    # new PR must be in fork for labels to actually match
+    with r2, fork2:
+        # branch in fork has no owner prefix, but HEAD for cross-repo PR does
+        fork2.make_commits("b", Commit('b', tree={'b': 'b'}), ref=f'heads/{pr1_b_id.refname}')
+        pr2_b = r2.make_pr(title="b", target="b", head=pr1_b_id.label)
+        r2.post_status(pr2_b.head, 'success')
+    env.run_crons()
+
+    pr2_b_id = to_pr(env, pr2_b)
+    assert pr2_b_id.batch_id == pr1_b_id.batch_id
+    assert len(pr2_b_id.forwardport_ids) == 1, \
+        "since the batch is already forward ported, the new PR should" \
+         " immediately be forward ported to match"
+    assert pr2_b_id.forwardport_ids.label == pr1_c_id.label
+
+    pr2_a = r1.get_pr(pr1_b_id.number)
+    with r1, r2:
+        pr2_a.post_comment('hansen r+', config['role_reviewer']['token'])
+        pr2_b.post_comment("hansen r+", config['role_reviewer']['token'])
+    env.run_crons()
+
+    with r1, r2:
+        r1.post_status('staging.b', 'success')
+        r2.post_status('staging.b', 'success')
+    env.run_crons()
+
+    assert pr1_b_id.state == 'merged'
+    assert pr2_b_id.state == 'merged'
+
+    assert len(pr2_b_id.forwardport_ids) == 1,\
+        "verify that pr2_b did not get forward ported again on merge"
+    pr2_c = r2.get_pr(pr2_b_id.forwardport_ids.number)
+    assert pr2_c.comments == [
+        seen(env, pr2_c, users),
+        (users['user'], '''\
+@{user} this PR targets c and is the last of the forward-port chain.
+
+To merge the full chain, use
+> @hansen r+
+
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+'''.format_map(users)),
+    ]
+
+def test_add_to_forward_port_conflict(env, config, make_repo, users):
+    """If a PR is added to an existing forward port sequence, and it causes
+    conflicts when forward ported, it should be treated similarly to an *update*
+    causing a conflict: the PR is still created, but it's set in conflict.
+    """
+    # region setup
+    r1, _ = make_basic(env, config, make_repo, statuses="default")
+    r2, fork2 = make_basic(env, config, make_repo, statuses="default")
+    project = env['runbot_merge.project'].search([])
+    with r2:
+        r2.make_commits(
+            "c",
+            Commit("C-onflict", tree={"b": "X"}),
+            ref="heads/c"
+        )
+
+    with r1:
+        r1.make_commits('a', Commit('a', tree={'a': 'a'}), ref="heads/pr1")
+        pr1_a = r1.make_pr(target="a", head="pr1")
+        r1.post_status(pr1_a.head, 'success')
+        pr1_a.post_comment('hansen r+', config['role_reviewer']['token'])
+    env.run_crons()
+    with r1, r2:
+        r1.post_status('staging.a', 'success')
+        r2.post_status('staging.a', 'success')
+    env.run_crons()
+
+    # region port forward
+    pr1_a_id = to_pr(env, pr1_a)
+    pr1_b_id = pr1_a_id.forwardport_ids
+    assert pr1_b_id
+    with r1:
+        r1.post_status(pr1_b_id.head, 'success')
+    env.run_crons()
+    pr1_c_id = pr1_a_id.forwardport_ids - pr1_b_id
+    assert pr1_c_id
+    # endregion
+    # endregion
+
+    # new PR must be in fork for labels to actually match
+    with r2, fork2:
+        # branch in fork has no owner prefix, but HEAD for cross-repo PR does
+        fork2.make_commits("b", Commit('b', tree={'b': 'b'}), ref=f'heads/{pr1_b_id.refname}')
+        pr2_b = r2.make_pr(title="b", target="b", head=pr1_b_id.label)
+        r2.post_status(pr2_b.head, 'success')
+    env.run_crons()
+
+    pr2_b_id = to_pr(env, pr2_b)
+    assert pr2_b_id.batch_id == pr1_b_id.batch_id
+    pr2_c_id = pr2_b_id.forwardport_ids
+    assert len(pr2_c_id) == 1, \
+        "since the batch is already forward ported, the new PR should" \
+         " immediately be forward ported to match"
+    assert pr2_c_id.label == pr1_c_id.label
+    assert not pr2_c_id.parent_id, "conflict -> should be detached"
+    assert pr2_c_id.detach_reason
+
+    pr2_a = r1.get_pr(pr1_b_id.number)
+    with r1, r2:
+        pr2_a.post_comment('hansen r+', config['role_reviewer']['token'])
+        pr2_b.post_comment("hansen r+", config['role_reviewer']['token'])
+    env.run_crons()
+
+    with r1, r2:
+        r1.post_status('staging.b', 'success')
+        r2.post_status('staging.b', 'success')
+    env.run_crons()
+
+    assert pr1_b_id.state == 'merged'
+    assert pr2_b_id.state == 'merged'
+
+    pr2_c = r2.get_pr(pr2_c_id.number)
+    assert pr2_c.comments == [
+        seen(env, pr2_c, users),
+        # should have conflicts
+        (users['user'], re_matches(r"""@{user} cherrypicking of pull request {previous.display_name} failed\.
+
+stdout:
+```
+Auto-merging b
+CONFLICT \(add/add\): Merge conflict in b
+
+```
+
+stderr:
+```
+.*
+```
+
+Either perform the forward-port manually \(and push to this branch, proceeding as usual\) or close this PR \(maybe\?\)\.
+
+In the former case, you may want to edit this PR message as well\.
+
+:warning: after resolving this conflict, you will need to merge it via @{project.github_prefix}\.
+
+More info at https://github\.com/odoo/odoo/wiki/Mergebot#forward-port
+""".format(project=project, previous=pr2_b_id, **users), re.DOTALL))
+    ]
diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index f72b286a..b00bf520 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -49,8 +49,11 @@ class re_matches:
     def __eq__(self, text):
         return self._r.match(text)
 
+    def __str__(self):
+        return re.sub(r'\\(.)', r'\1', self._r.pattern)
+
     def __repr__(self):
-        return repr(self._r.pattern)
+        return repr(str(self))
 
 def seen(env, pr, users):
     return users['user'], f'[Pull request status dashboard]({to_pr(env, pr).url}).'
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index e689468b..0a7822d4 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -82,13 +82,13 @@ class Batch(models.Model):
         # in DB both will prefix-match on the literal prefix then apply a
         # trivial filter (even though the filter is technically unnecessary for
         # the first form), doing it like this means we don't have to `- self`
-        # in the `not includ_self` case
+        # in the ``not include_self`` case
         if include_self:
             pattern = self.parent_path + '%'
         else:
             pattern = self.parent_path + '_%'
 
-        return self.search([("parent_path", '=like', pattern)])
+        return self.search([("parent_path", '=like', pattern)], order="parent_path")
 
     # also depends on all the descendants of the source or sth
     @api.depends('parent_path')
@@ -323,9 +323,10 @@ class Batch(models.Model):
                 'source_id': source.id,
                 # only link to previous PR of sequence if cherrypick passed
                 'parent_id': pr.id if not has_conflicts else False,
-                'detach_reason': "conflicts: {}".format(
-                    f'\n{conflicts[pr]}\n{conflicts[pr]}'.strip()
-                ) if has_conflicts else None,
+                'detach_reason': "conflicts:\n{}".format('\n\n'.join(
+                    f"{out}\n{err}".strip()
+                    for _, out, err, _ in filter(None, conflicts.values())
+                )) if has_conflicts else None,
             })
             if has_conflicts and pr.parent_id and pr.state not in ('merged', 'closed'):
                 self.env.ref('runbot_merge.forwardport.failure.conflict')._send(
@@ -336,63 +337,7 @@ class Batch(models.Model):
                 )
 
         for pr, new_pr in zip(prs, new_batch):
-            (h, out, err, hh) = conflicts.get(pr) or (None, None, None, None)
-
-            if h:
-                sout = serr = ''
-                if out.strip():
-                    sout = f"\nstdout:\n```\n{out}\n```\n"
-                if err.strip():
-                    serr = f"\nstderr:\n```\n{err}\n```\n"
-
-                lines = ''
-                if len(hh) > 1:
-                    lines = '\n' + ''.join(
-                        '* %s%s\n' % (sha, ' <- on this commit' if sha == h else '')
-                        for sha in hh
-                    )
-                template = 'runbot_merge.forwardport.failure'
-                format_args = {
-                    'pr': new_pr,
-                    'commits': lines,
-                    'stdout': sout,
-                    'stderr': serr,
-                    'footer': FOOTER,
-                }
-            elif has_conflicts:
-                template = 'runbot_merge.forwardport.linked'
-                format_args = {
-                    'pr': new_pr,
-                    'siblings': ', '.join(p.display_name for p in (new_batch - new_pr)),
-                    'footer': FOOTER,
-                }
-            elif not new_pr._find_next_target():
-                ancestors = "".join(
-                    f"* {p.display_name}\n"
-                    for p in pr._iter_ancestors()
-                    if p.parent_id
-                    if p.state not in ('closed', 'merged')
-                    if p.target.active
-                )
-                template = 'runbot_merge.forwardport.final'
-                format_args = {
-                    'pr': new_pr,
-                    'containing': ' containing:' if ancestors else '.',
-                    'ancestors': ancestors,
-                    'footer': FOOTER,
-                }
-            else:
-                template = 'runbot_merge.forwardport.intermediate'
-                format_args = {
-                    'pr': new_pr,
-                    'footer': FOOTER,
-                }
-            self.env.ref(template)._send(
-                repository=new_pr.repository,
-                pull_request=new_pr.number,
-                token_field='fp_github_token',
-                format_args=format_args,
-            )
+            new_pr._fp_conflict_feedback(pr, conflicts)
 
             labels = ['forwardport']
             if has_conflicts:
@@ -409,7 +354,6 @@ class Batch(models.Model):
         new_batch._schedule_fp_followup()
         return new_batch
 
-
     def _schedule_fp_followup(self):
         _logger = logging.getLogger(__name__).getChild('forwardport.next')
         # if the PR has a parent and is CI-validated, enqueue the next PR
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index ee6b76e5..edbecd18 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -26,6 +26,7 @@ from .utils import enum
 from .. import github, exceptions, controllers, utils
 
 _logger = logging.getLogger(__name__)
+FOOTER = '\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n'
 
 
 class StatusConfiguration(models.Model):
@@ -1386,6 +1387,64 @@ class PullRequests(models.Model):
 
         return True
 
+    def _fp_conflict_feedback(self, previous_pr, conflicts):
+        (h, out, err, hh) = conflicts.get(previous_pr) or (None, None, None, None)
+        if h:
+            sout = serr = ''
+            if out.strip():
+                sout = f"\nstdout:\n```\n{out}\n```\n"
+            if err.strip():
+                serr = f"\nstderr:\n```\n{err}\n```\n"
+
+            lines = ''
+            if len(hh) > 1:
+                lines = '\n' + ''.join(
+                    '* %s%s\n' % (sha, ' <- on this commit' if sha == h else '')
+                    for sha in hh
+                )
+            template = 'runbot_merge.forwardport.failure'
+            format_args = {
+                'pr': self,
+                'commits': lines,
+                'stdout': sout,
+                'stderr': serr,
+                'footer': FOOTER,
+            }
+        elif any(conflicts.values()):
+            template = 'runbot_merge.forwardport.linked'
+            format_args = {
+                'pr': self,
+                'siblings': ', '.join(p.display_name for p in (self.batch_id - self)),
+                'footer': FOOTER,
+            }
+        elif not self._find_next_target():
+            ancestors = "".join(
+                f"* {p.display_name}\n"
+                for p in previous_pr._iter_ancestors()
+                if p.parent_id
+                if p.state not in ('closed', 'merged')
+                if p.target.active
+            )
+            template = 'runbot_merge.forwardport.final'
+            format_args = {
+                'pr': self,
+                'containing': ' containing:' if ancestors else '.',
+                'ancestors': ancestors,
+                'footer': FOOTER,
+            }
+        else:
+            template = 'runbot_merge.forwardport.intermediate'
+            format_args = {
+                'pr': self,
+                'footer': FOOTER,
+            }
+        self.env.ref(template)._send(
+            repository=self.repository,
+            pull_request=self.number,
+            token_field='fp_github_token',
+            format_args=format_args,
+        )
+
 # ordering is a bit unintuitive because the lowest sequence (and name)
 # is the last link of the fp chain, reasoning is a bit more natural the
 # other way around (highest object is the last), especially with Python

From 0e0348e4df9e8aae73170db03e5e1b407e7dca9f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 17 May 2024 10:51:16 +0200
Subject: [PATCH 100/192] [IMP] runbot_merge: preserve batch ordering in
 stagings

Batch ordering in stagings is important in order to correctly
reconstitute the full project history.

In the old mergebot, since batches are created on the fly during
staging this information is reified by the batch ids. But since batch
ids are now persistent and there is no relationship between the
creation of a batch and its merging (especially not relative to other
batches) it's an issue as reconstituting sub-staging git history would
be impossible.

Which is not the worst, but is not great.

The solution is to reify the join table between stagings and batches
in order for *that* to keep history (simply via the sequential PK),
and in converting to the new system carefully generate the new links
in an order matching the old batch ids.
---
 runbot_merge/models/batch.py              | 39 ++++++++++++++++++++++-
 runbot_merge/models/pull_requests.py      |  8 ++++-
 runbot_merge/models/stagings_create.py    |  5 ++-
 runbot_merge/security/ir.model.access.csv |  1 +
 runbot_merge/tests/test_basic.py          |  9 ++++++
 5 files changed, 57 insertions(+), 5 deletions(-)

diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index 0a7822d4..e48f0541 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -9,6 +9,7 @@ from collections import defaultdict
 from collections.abc import Iterator
 
 import requests
+from psycopg2 import sql
 
 from odoo import models, fields, api
 from .utils import enum
@@ -18,6 +19,28 @@ _logger = logging.getLogger(__name__)
 FOOTER = '\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n'
 
 
+class StagingBatch(models.Model):
+    _name = 'runbot_merge.staging.batch'
+    _description = "link between batches and staging in order to maintain an " \
+                   "ordering relationship between the batches of a staging"
+    _log_access = False
+    _order = 'id'
+
+    runbot_merge_batch_id = fields.Many2one('runbot_merge.batch', required=True)
+    runbot_merge_stagings_id = fields.Many2one('runbot_merge.stagings', required=True)
+
+    def init(self):
+        super().init()
+
+        self.env.cr.execute(sql.SQL("""
+        CREATE UNIQUE INDEX IF NOT EXISTS runbot_merge_staging_batch_idx
+            ON {table} (runbot_merge_stagings_id, runbot_merge_batch_id);
+
+        CREATE INDEX IF NOT EXISTS runbot_merge_staging_batch_rev
+            ON {table} (runbot_merge_batch_id) INCLUDE (runbot_merge_stagings_id);
+        """).format(table=sql.Identifier(self._table)))
+
+
 class Batch(models.Model):
     """ A batch is a "horizontal" grouping of *codependent* PRs: PRs with
     the same label & target but for different repositories. These are
@@ -31,7 +54,12 @@ class Batch(models.Model):
     _parent_store = True
 
     target = fields.Many2one('runbot_merge.branch', store=True, compute='_compute_target')
-    staging_ids = fields.Many2many('runbot_merge.stagings')
+    batch_staging_ids = fields.One2many('runbot_merge.staging.batch', 'runbot_merge_batch_id')
+    staging_ids = fields.Many2many(
+        'runbot_merge.stagings',
+        compute="_compute_staging_ids",
+        context={'active_test': False},
+    )
     split_id = fields.Many2one('runbot_merge.split', index=True)
 
     prs = fields.One2many('runbot_merge.pull_requests', 'batch_id')
@@ -74,6 +102,11 @@ class Batch(models.Model):
         context={"active_test": False},
     )
 
+    @api.depends('batch_staging_ids.runbot_merge_stagings_id')
+    def _compute_staging_ids(self):
+        for batch in self:
+            batch.staging_ids = batch.batch_staging_ids.runbot_merge_stagings_id
+
     @property
     def source(self):
         return self.browse(map(int, self.parent_path.split('/', 1)[:1]))
@@ -452,6 +485,10 @@ class Batch(models.Model):
 
         return True
 
+    @api.ondelete(at_uninstall=True)
+    def _on_delete_clear_stagings(self):
+        self.batch_staging_ids.unlink()
+
     def unlink(self):
         """
         batches can be unlinked if they:
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index edbecd18..b07b031c 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1763,7 +1763,8 @@ class Stagings(models.Model):
 
     target = fields.Many2one('runbot_merge.branch', required=True, index=True)
 
-    batch_ids = fields.Many2many('runbot_merge.batch', context={'active_test': False})
+    staging_batch_ids = fields.One2many('runbot_merge.staging.batch', 'runbot_merge_stagings_id')
+    batch_ids = fields.Many2many('runbot_merge.batch', context={'active_test': False}, compute="_compute_batch_ids")
     pr_ids = fields.One2many('runbot_merge.pull_requests', compute='_compute_prs')
     state = fields.Selection([
         ('success', 'Success'),
@@ -1814,6 +1815,11 @@ class Stagings(models.Model):
             for staging in self
         ]
 
+    @api.depends('staging_batch_ids.runbot_merge_batch_id')
+    def _compute_batch_ids(self):
+        for staging in self:
+            staging.batch_ids = staging.staging_batch_ids.runbot_merge_batch_id
+
     @api.depends('heads')
     def _compute_statuses(self):
         """ Fetches statuses associated with the various heads, returned as
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index 017b63a9..1f620154 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -8,13 +8,12 @@ import os
 import re
 from collections.abc import Mapping
 from difflib import Differ
-from itertools import takewhile
 from operator import itemgetter
 from typing import Dict, Union, Optional, Literal, Callable, Iterator, Tuple, List, TypeAlias
 
 from werkzeug.datastructures import Headers
 
-from odoo import api, models, fields
+from odoo import api, models, fields, Command
 from odoo.tools import OrderedSet, groupby
 from .pull_requests import Branch, Stagings, PullRequests, Repository
 from .batch import Batch
@@ -166,7 +165,7 @@ For-Commit-Id: {it.head}
     # create actual staging object
     st: Stagings = env['runbot_merge.stagings'].create({
         'target': branch.id,
-        'batch_ids': [(4, batch.id, 0) for batch in staged],
+        'staging_batch_ids': [Command.create({'runbot_merge_batch_id': batch.id}) for batch in staged],
         'heads': heads,
         'commits': commits,
     })
diff --git a/runbot_merge/security/ir.model.access.csv b/runbot_merge/security/ir.model.access.csv
index 5b2e102b..28f2bd7e 100644
--- a/runbot_merge/security/ir.model.access.csv
+++ b/runbot_merge/security/ir.model.access.csv
@@ -16,6 +16,7 @@ access_runbot_merge_stagings_commits_admin,Admin access to staging commits,model
 access_runbot_merge_stagings_cancel_admin,Admin access to cancelling stagings,model_runbot_merge_stagings_cancel,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_split_admin,Admin access to splits,model_runbot_merge_split,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_batch_admin,Admin access to batches,model_runbot_merge_batch,runbot_merge.group_admin,1,1,1,1
+access_runbot_merge_staging_batch_admin,Admin access to batch/staging link,model_runbot_merge_staging_batch,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_fetch_job_admin,Admin access to fetch jobs,model_runbot_merge_fetch_job,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_pull_requests_feedback_admin,Admin access to feedback,model_runbot_merge_pull_requests_feedback,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_review_rights,Admin access to review permissions,model_res_partner_review,runbot_merge.group_admin,1,1,1,1
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 77fb3859..e47ff180 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -2784,6 +2784,11 @@ class TestBatching(object):
 
         staging = ensure_one(env['runbot_merge.stagings'].search([]))
         assert staging.pr_ids == pr11 | pr12 | pr21
+        assert list(staging.batch_ids) == [
+            pr11.batch_id,
+            pr12.batch_id,
+            pr21.batch_id,
+        ]
         assert not pr22.staging_id
 
     def test_batching_urgent(self, env, repo, config):
@@ -2804,6 +2809,10 @@ class TestBatching(object):
         staging_1 = sm_all.staging_id
         assert staging_1
         assert len(staging_1) == 1
+        assert list(staging_1.batch_ids) == [
+            p_11.batch_id,
+            p_12.batch_id,
+        ]
 
         # no statuses run on PR0s
         with repo:

From bbce5f8f46f270099e54b4a39c4e548798cabfd0 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 3 Apr 2024 12:08:04 +0200
Subject: [PATCH 101/192] [IMP] *: don't remove PRs from batches on close

Initially wanted to skip this only for FW PRs, but after some thinking
I feel this info could still be valuable even for non-fw PRs which
were never merged in the first place.

Requires a few adjustments to not break *everything*: `batch.prs`
excludes closed PRs by default as most processes only expect to be
faced by a closed PR inside a batch, and we *especially* want to avoid
that before the batch is merged (as we'd risk staging a closed PR).

However since PRs don't get removed from batches anymore (and batches
don't get deleted when they have no PRs) we now may have a bunch of
batches whose PRs (usually a single one) are all closed, this has two
major side-effects:

- a new PR may get attached to an old batch full of closed PRs (as
  batches are filtered out on being *merged*), which is weird
- the eventual list of batches gets polluted with a bunch of
  irrelevant batches which are hard to filter out

The solution is to reintroduce an `active` field, as a stored compute
field based on the state of batch PRs. This way if all PRs of a batch
are closed it switches to inactive, and is automatically filtered out
by search which solves both issues.
---
 forwardport/tests/test_batches.py             |  21 +++-
 forwardport/tests/test_weird.py               |  12 +-
 .../migrations/15.0.1.12/pre-migration.py     |   6 +-
 runbot_merge/models/batch.py                  |  37 +++++--
 runbot_merge/models/pull_requests.py          |  22 ++--
 runbot_merge/tests/test_batch_consistency.py  | 104 ++++++++++++++++++
 runbot_merge/tests/test_multirepo.py          |  38 ++++++-
 7 files changed, 208 insertions(+), 32 deletions(-)
 create mode 100644 runbot_merge/tests/test_batch_consistency.py

diff --git a/forwardport/tests/test_batches.py b/forwardport/tests/test_batches.py
index 0ada435c..ad1c6f8b 100644
--- a/forwardport/tests/test_batches.py
+++ b/forwardport/tests/test_batches.py
@@ -1,7 +1,5 @@
 import re
 
-import pytest
-
 from utils import Commit, make_basic, to_pr, seen, re_matches
 
 
@@ -133,6 +131,25 @@ def test_closing_during_fp(env, config, make_repo, users):
         "only one of the forward ports should be ported"
     assert not env['runbot_merge.pull_requests'].search([('parent_id', '=', pr1_1_id.id)]),\
         "the closed PR should not be ported"
+    assert env['runbot_merge.pull_requests'].search([('source_id', '=', pr1_id.id)]) == pr1_1_id,\
+        "the closed PR should not be ported"
+
+    r1_b_head = r1.commit("b")
+    with r2:
+        r2.get_pr(pr2_1_id.number).post_comment('hansen r+', config['role_reviewer']['token'])
+    env.run_crons()
+    assert not pr2_1_id.blocked
+    assert not pr2_1_id.batch_id.blocked
+    st = pr2_1_id.staging_id
+    assert st
+    with r1, r2:
+        r1.post_status('staging.b', 'success')
+        r2.post_status('staging.b', 'success')
+    env.run_crons()
+    assert st.state == 'success'
+
+    assert r1_b_head.id == r1.commit("b").id, \
+        "r1:b's head should not have been touched"
 
 def test_add_pr_during_fp(env, config, make_repo, users):
     """ It should be possible to add new PRs to an FP batch
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index 704d5c2a..74862760 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -1085,6 +1085,7 @@ def test_maintain_batch_history(env, config, make_repo, users):
     assert pr1_c_id.batch_id.parent_id == pr1_b_id.batch_id
     assert pr1_b_id.batch_id.parent_id == pr1_a_id.batch_id
     b_batch = pr1_b_id.batch_id
+    assert b_batch
     # endregion
 
     pr1_b = repo.get_pr(pr1_b_id.number)
@@ -1094,10 +1095,11 @@ def test_maintain_batch_history(env, config, make_repo, users):
     assert pr1_b_id.state == 'closed'
 
     # region check that all the batches are *still* set up correctly
-    assert pr1_a_id.batch_id
-    assert not pr1_b_id.batch_id, "the PR still gets detached from the batch"
-    assert b_batch.exists(), "the batch is not deleted tho"
-    assert pr1_c_id.batch_id
+    assert b_batch.exists()
+    assert pr1_a_id.batch_id == b_batch.parent_id
+    assert pr1_b_id.batch_id == b_batch
     assert pr1_c_id.batch_id.parent_id == b_batch
-    assert b_batch.parent_id == pr1_a_id.batch_id
+
+    assert pr1_b_id in b_batch.all_prs, "the PR is still in the batch"
+    assert pr1_b_id not in b_batch.prs, "the PR is not in the open/active batch PRs"
     # endregion
diff --git a/runbot_merge/migrations/15.0.1.12/pre-migration.py b/runbot_merge/migrations/15.0.1.12/pre-migration.py
index 12762afe..107c624b 100644
--- a/runbot_merge/migrations/15.0.1.12/pre-migration.py
+++ b/runbot_merge/migrations/15.0.1.12/pre-migration.py
@@ -76,7 +76,11 @@ def migrate(cr, version):
     # avoid SQL taking absolutely ungodly amounts of time
     cr.execute("SET statement_timeout = '60s'")
     # will be recreated & computed on the fly
-    cr.execute("ALTER TABLE runbot_merge_batch DROP COLUMN target")
+    cr.execute("""
+    ALTER TABLE runbot_merge_batch
+        DROP COLUMN target,
+        DROP COLUMN active
+    """)
 
     cleanup(cr)
 
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index e48f0541..1472ccd1 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -62,7 +62,9 @@ class Batch(models.Model):
     )
     split_id = fields.Many2one('runbot_merge.split', index=True)
 
-    prs = fields.One2many('runbot_merge.pull_requests', 'batch_id')
+    all_prs = fields.One2many('runbot_merge.pull_requests', 'batch_id')
+    prs = fields.One2many('runbot_merge.pull_requests', compute='_compute_open_prs', search='_search_open_prs')
+    active = fields.Boolean(compute='_compute_active', store=True, help="closed batches (batches containing only closed PRs)")
 
     fw_policy = fields.Selection([
         ('default', "Default"),
@@ -121,14 +123,20 @@ class Batch(models.Model):
         else:
             pattern = self.parent_path + '_%'
 
-        return self.search([("parent_path", '=like', pattern)], order="parent_path")
+        act = self.env.context.get('active_test', True)
+        return self\
+            .with_context(active_test=False)\
+            .search([("parent_path", '=like', pattern)], order="parent_path")\
+            .with_context(active_test=act)
 
     # also depends on all the descendants of the source or sth
     @api.depends('parent_path')
     def _compute_genealogy(self):
         for batch in self:
             sid = next(iter(batch.parent_path.split('/', 1)))
-            batch.genealogy_ids = self.search([("parent_path", "=like", f"{sid}/%")], order="parent_path")
+            batch.genealogy_ids = self \
+                .with_context(active_test=False)\
+                .search([("parent_path", "=like", f"{sid}/%")], order="parent_path")\
 
     def _auto_init(self):
         for field in self._fields.values():
@@ -155,15 +163,28 @@ class Batch(models.Model):
         WHERE parent_id IS NOT NULL;
         """)
 
-    @api.depends("prs.target")
+    @api.depends('all_prs.closed')
+    def _compute_active(self):
+        for b in self:
+            b.active = not all(p.closed for p in b.all_prs)
+
+    @api.depends('all_prs.closed')
+    def _compute_open_prs(self):
+        for b in self:
+            b.prs = b.all_prs.filtered(lambda p: not p.closed)
+
+    def _search_open_prs(self, operator, value):
+        return [('all_prs', operator, value), ('active', '=', True)]
+
+    @api.depends("all_prs.target")
     def _compute_target(self):
         for batch in self:
             if len(batch.prs) == 1:
-                batch.target = batch.prs.target
+                batch.target = batch.all_prs.target
             else:
-                targets = set(batch.prs.mapped('target'))
+                targets = set(batch.all_prs.mapped('target'))
                 if not targets:
-                    targets = set(batch.prs.mapped('target'))
+                    targets = set(batch.all_prs.mapped('target'))
                 if len(targets) == 1:
                     batch.target = targets.pop()
                 else:
@@ -180,6 +201,8 @@ class Batch(models.Model):
         for batch in self:
             if batch.merge_date:
                 batch.blocked = "Merged."
+            elif not batch.active:
+                batch.blocked = "all prs are closed"
             elif blocking := batch.prs.filtered(
                 lambda p: p.error or p.draft or not (p.squash or p.merge_method)
             ):
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index b07b031c..3e706268 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -563,11 +563,6 @@ class PullRequests(models.Model):
             return json.loads(self.overrides)
         return {}
 
-    @api.depends('batch_ids.active')
-    def _compute_active_batch(self):
-        for r in self:
-            r.batch_id = r.batch_ids.filtered(lambda b: b.active)[:1]
-
     def _get_or_schedule(self, repo_name, number, *, target=None, closing=False):
         repo = self.env['runbot_merge.repository'].search([('name', '=', repo_name)])
         if not repo:
@@ -1255,9 +1250,7 @@ class PullRequests(models.Model):
         match vals.get('closed'):
             case True if not self.closed:
                 vals['reviewed_by'] = False
-                vals['batch_id'] = False
-                self.env.cr.precommit.add(self.batch_id.unlink)
-            case False if self.closed:
+            case False if self.closed and not self.batch_id:
                 vals['batch_id'] = self._get_batch(
                     target=vals.get('target') or self.target.id,
                     label=vals.get('label') or self.label,
@@ -1382,8 +1375,7 @@ class PullRequests(models.Model):
             return False
 
         self.unstage("closed by %s", by)
-        # `write` automatically unsets reviewer & batch when closing but...
-        self.write({'closed': True, 'reviewed_by': False, 'batch_id': False})
+        self.write({'closed': True, 'reviewed_by': False})
 
         return True
 
@@ -1764,7 +1756,12 @@ class Stagings(models.Model):
     target = fields.Many2one('runbot_merge.branch', required=True, index=True)
 
     staging_batch_ids = fields.One2many('runbot_merge.staging.batch', 'runbot_merge_stagings_id')
-    batch_ids = fields.Many2many('runbot_merge.batch', context={'active_test': False}, compute="_compute_batch_ids")
+    batch_ids = fields.Many2many(
+        'runbot_merge.batch',
+        context={'active_test': False},
+        compute="_compute_batch_ids",
+        search="_search_batch_ids",
+    )
     pr_ids = fields.One2many('runbot_merge.pull_requests', compute='_compute_prs')
     state = fields.Selection([
         ('success', 'Success'),
@@ -1820,6 +1817,9 @@ class Stagings(models.Model):
         for staging in self:
             staging.batch_ids = staging.staging_batch_ids.runbot_merge_batch_id
 
+    def _search_batch_ids(self, operator, value):
+        return [('staging_batch_ids.runbot_merge_batch_id', operator, value)]
+
     @api.depends('heads')
     def _compute_statuses(self):
         """ Fetches statuses associated with the various heads, returned as
diff --git a/runbot_merge/tests/test_batch_consistency.py b/runbot_merge/tests/test_batch_consistency.py
new file mode 100644
index 00000000..ef3c88f6
--- /dev/null
+++ b/runbot_merge/tests/test_batch_consistency.py
@@ -0,0 +1,104 @@
+"""This module tests edge cases specific to the batch objects themselves,
+without wider relevance and thus other location.
+"""
+from utils import Commit, to_pr
+
+
+def test_close_single(env, project, make_repo, setreviewers):
+    """If a batch has a single PR and that PR gets closed, the batch should be
+    inactive *and* blocked.
+    """
+    repo = make_repo('wheee')
+    r = env['runbot_merge.repository'].create({
+        'project_id': project.id,
+        'name': repo.name,
+        'required_statuses': 'default',
+        'group_id': False,
+    })
+    setreviewers(r)
+
+    with repo:
+        repo.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
+        [c] = repo.make_commits('master', Commit('b', tree={"b": "b"}))
+        pr = repo.make_pr(head=c, target='master')
+    env.run_crons()
+
+    pr_id = to_pr(env, pr)
+    batch_id = pr_id.batch_id
+    assert pr_id.state == 'opened'
+    assert batch_id.blocked
+    Batches = env['runbot_merge.batch']
+    assert Batches.search_count([]) == 1
+
+    with repo:
+        pr.close()
+
+    assert pr_id.state == 'closed'
+    assert batch_id.all_prs == pr_id
+    assert batch_id.prs == pr_id.browse(())
+    assert batch_id.blocked == "all prs are closed"
+    assert not batch_id.active
+
+    assert Batches.search_count([]) == 0
+
+def test_close_multiple(env, project, make_repo, setreviewers):
+    """If a batch has a single PR and that PR gets closed, the batch should be
+    inactive *and* blocked.
+    """
+    Batches = env['runbot_merge.batch']
+    repo1 = make_repo('wheee')
+    repo2 = make_repo('wheeee')
+    project.write({
+        'repo_ids': [(0, 0, {
+            'name': repo1.name,
+            'required_statuses': 'default',
+            'group_id': False,
+        }), (0, 0, {
+            'name': repo2.name,
+            'required_statuses': 'default',
+            'group_id': False,
+        })]
+    })
+    setreviewers(*project.repo_ids)
+
+    with repo1:
+        repo1.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
+        repo1.make_commits('master', Commit('b', tree={"b": "b"}), ref='heads/a_pr')
+        pr1 = repo1.make_pr(head='a_pr', target='master')
+
+    with repo2:
+        repo2.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
+        repo2.make_commits('master', Commit('b', tree={"b": "b"}), ref='heads/a_pr')
+        pr2 = repo2.make_pr(head='a_pr', target='master')
+
+    pr1_id = to_pr(env, pr1)
+    pr2_id = to_pr(env, pr2)
+    batch_id = pr1_id.batch_id
+    assert pr2_id.batch_id == batch_id
+
+    assert pr1_id.state == 'opened'
+    assert pr2_id.state == 'opened'
+    assert batch_id.all_prs == pr1_id | pr2_id
+    assert batch_id.prs == pr1_id | pr2_id
+    assert batch_id.active
+    assert Batches.search_count([]) == 1
+
+    with repo1:
+        pr1.close()
+
+    assert pr1_id.state == 'closed'
+    assert pr2_id.state == 'opened'
+    assert batch_id.all_prs == pr1_id | pr2_id
+    assert batch_id.prs == pr2_id
+    assert batch_id.active
+    assert Batches.search_count([]) == 1
+
+    with repo2:
+        pr2.close()
+
+    assert pr1_id.state == 'closed'
+    assert pr2_id.state == 'closed'
+    assert batch_id.all_prs == pr1_id | pr2_id
+    assert batch_id.prs == env['runbot_merge.pull_requests'].browse(())
+    assert not batch_id.active
+    assert Batches.search_count([]) == 0
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 1f09b817..873986dc 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -811,14 +811,40 @@ class TestBlocked:
             repo_a.make_commits(None, Commit('initial', tree={'a0': 'a'}), ref='heads/master')
             repo_b.make_commits(None, Commit('initial', tree={'b0': 'b'}), ref='heads/master')
 
-            pr = make_pr(repo_a, 'xxx', [{'a1': 'a'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'],)
-            b = make_pr(repo_b, 'xxx', [{'b1': 'b'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'], statuses=[])
+            pr1_a = make_pr(repo_a, 'xxx', [{'a1': 'a'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'],)
+            pr1_b = make_pr(repo_b, 'xxx', [{'b1': 'b'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'], statuses=[])
         env.run_crons()
 
-        p = to_pr(env, pr)
-        assert p.blocked
-        with repo_b: b.close()
-        assert not p.blocked
+        head_a = repo_a.commit('master').id
+        head_b = repo_b.commit('master').id
+        pr1_a_id = to_pr(env, pr1_a)
+        pr1_b_id = to_pr(env, pr1_b)
+        assert pr1_a_id.blocked
+        with repo_b: pr1_b.close()
+        assert not pr1_a_id.blocked
+        assert len(pr1_a_id.batch_id.all_prs) == 2
+        assert pr1_a_id.state == 'ready'
+        assert pr1_b_id.state == 'closed'
+        env.run_crons()
+        assert pr1_a_id.staging_id
+        with repo_a, repo_b:
+            repo_a.post_status('staging.master', 'success')
+            repo_b.post_status('staging.master', 'success')
+        env.run_crons()
+        assert pr1_a_id.state == 'merged'
+        assert pr1_a_id.batch_id.merge_date
+        assert repo_a.commit('master').id != head_a, \
+            "the master of repo A should be updated"
+        assert repo_b.commit('master').id == head_b, \
+            "the master of repo B should not be updated"
+
+        with repo_a:
+            pr2_a = make_pr(repo_a, "xxx", [{'x': 'x'}], user=config['role_user']['token'], reviewer=config['role_reviewer']['token'])
+        env.run_crons()
+        pr2_a_id = to_pr(env, pr2_a)
+        assert pr2_a_id.batch_id != pr1_a_id.batch_id
+        assert pr2_a_id.label == pr1_a_id.label
+        assert len(pr2_a_id.batch_id.all_prs) == 1
 
     def test_linked_merged(self, env, repo_a, repo_b, config):
         with repo_a, repo_b:

From 3191c44459c094ce5659dec9cb5739eb8a4c07ef Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 22 Apr 2024 14:11:38 +0200
Subject: [PATCH 102/192] [ADD] runbot_merge: synthetic batches & stagings to
 freeze wizard

Merged PRs should have a batch which should have a staging, this makes
the treatment uniform across the board and avoids funky data which is
hard to place or issues when reconstructing history.

Also create synthetic batches & stagings for older freezes (and bumps)
---
 .../migrations/15.0.1.12/pre-migration.py     | 117 ++++++++++++++++--
 .../models/project_freeze/__init__.py         |  70 +++++++++++
 runbot_merge/tests/test_multirepo.py          |  10 ++
 3 files changed, 189 insertions(+), 8 deletions(-)

diff --git a/runbot_merge/migrations/15.0.1.12/pre-migration.py b/runbot_merge/migrations/15.0.1.12/pre-migration.py
index 107c624b..efcc27af 100644
--- a/runbot_merge/migrations/15.0.1.12/pre-migration.py
+++ b/runbot_merge/migrations/15.0.1.12/pre-migration.py
@@ -70,8 +70,114 @@ def hlink(url):
 def link(label, url):
     return f"{hlink(url)}{label}{hlink('')}"
 
+
+def batch_freezes(cr):
+    """Old freezes were created batch-less but marked as merged, to make things
+    more consistent and avoid losing them for e.g. synthetic git histories,
+    associate then with synthetic successful stagings
+    """
+    cr.execute("SELECT id FROM res_users WHERE login = 'moc@odoo.com'")
+    [uid] = cr.fetchone()
+    cr.execute("""
+    SELECT
+        array_agg(DISTINCT p.target) AS target,
+        array_agg(DISTINCT p.merge_date) AS merge_date,
+        json_object_agg(r.id, json_build_object(
+            'id', p.id,
+            'head', p.commits_map::json->''
+        )) AS prs
+
+    FROM runbot_merge_pull_requests p
+    JOIN runbot_merge_repository r ON (r.id = p.repository)
+    JOIN runbot_merge_branch t ON (t.id = p.target)
+
+    LEFT JOIN runbot_merge_batch_runbot_merge_pull_requests_rel bp ON (runbot_merge_pull_requests_id = p.id)
+    LEFT JOIN runbot_merge_batch b ON (runbot_merge_batch_id = b.id)
+    LEFT JOIN runbot_merge_stagings s ON (b.staging_id = s.id)
+
+    WHERE p.state = 'merged'
+      AND runbot_merge_pull_requests_id IS NULL
+      AND p.id != 1
+
+    GROUP BY label;
+    """)
+    freeze_batches = [
+        (target, merge_date, {int(r): p for r, p in prs.items()})
+        for [target], [merge_date], prs in cr._obj
+    ]
+
+    stagings = []
+    for t, m, prs in freeze_batches:
+        # fetch the preceding successful staging on master
+        cr.execute("""
+            SELECT id
+            FROM runbot_merge_stagings
+            -- target 1 = master (so we want the last successful master staging before the freeze)
+            WHERE state = 'success' AND staged_at < %s AND target = 1
+            ORDER BY staged_at DESC
+            LIMIT 1
+        """, [m])
+        cr.execute("""
+            SELECT repository_id, commit_id
+            FROM runbot_merge_stagings_commits
+            WHERE staging_id = %s
+        """, cr.fetchone())
+        commits = dict(cr._obj)
+
+        cr.execute("""
+            INSERT INTO runbot_merge_stagings
+            (state, active, create_uid, write_uid, target, staged_at, create_date, write_date)
+            VALUES ('success', false, %s, %s, %s, %s, %s, %s)
+            RETURNING id
+        """, [uid, uid, t, m, m, m])
+        [[staging]] = cr.fetchall()
+        stagings.append(staging)
+
+        for repo, pr in prs.items():
+            if repo not in commits:
+                cr.execute("""
+                    INSERT INTO runbot_merge_commit (sha) VALUES (%s)
+                    ON CONFLICT (sha) DO UPDATE
+                        SET to_check = runbot_merge.to_check
+                    RETURNING id
+                """, [pr['head']])
+                [cid] = cr.fetchone()
+                commits[repo] = cid
+
+        for repo, commit in commits.items():
+            cr.execute("""
+                INSERT INTO runbot_merge_stagings_commits
+                    (staging_id, repository_id, commit_id)
+                VALUES (%s, %s, %s)
+            """, [staging, repo, commit])
+            cr.execute("""
+                INSERT INTO runbot_merge_stagings_heads
+                    (staging_id, repository_id, commit_id)
+                VALUES (%s, %s, %s)
+            """, [staging, repo, commit])
+
+    batches = []
+    for staging, (_, date, _) in zip(stagings, freeze_batches):
+        cr.execute("""
+            INSERT INTO runbot_merge_batch
+                (create_uid, write_uid, staging_id, create_date, write_date)
+            VALUES (%s, %s, %s, %s, %s)
+            RETURNING id
+        """, [uid, uid, staging, date, date])
+        [[batch]] = cr.fetchall()
+        batches.append(batch)
+
+    for batch, (_, _, prs) in zip(batches, freeze_batches):
+        for pr in prs.values():
+            cr.execute("""
+                INSERT INTO runbot_merge_batch_runbot_merge_pull_requests_rel
+                    (runbot_merge_batch_id, runbot_merge_pull_requests_id)
+                VALUES (%s, %s)
+            """, [batch, pr['id']])
+
+
 def migrate(cr, version):
-    cr.execute("select 1 from forwardport_batches")
+    cr.execute("select from forwardport_batches")
     assert not cr.rowcount, f"can't migrate the mergebot with enqueued forward ports (found {cr.rowcount})"
     # avoid SQL taking absolutely ungodly amounts of time
     cr.execute("SET statement_timeout = '60s'")
@@ -83,6 +189,7 @@ def migrate(cr, version):
     """)
 
     cleanup(cr)
+    batch_freezes(cr)
 
     cr.execute("""
     SELECT
@@ -218,17 +325,13 @@ def migrate(cr, version):
             cr.rowcount,
         )
 
-    # FIXME: fixup PRs marked as merged which don't actually have a batch / staging?
-
     # the relinked batches are those from stagings, but that means merged PRs
     # (or at least PRs we tried to merge), we also need batches for non-closed
     # non-merged PRs
-    logger.info("collect unbatched PRs PRs...")
+    logger.info("collect unbatched PRs...")
     cr.execute("""
     SELECT
         CASE
-            -- FIXME: should closed PRs w/o a batch be split out or matched with
-            --        one another?
             WHEN label SIMILAR TO '%%:patch-[[:digit:]]+'
                 THEN id::text
             ELSE label
@@ -272,8 +375,6 @@ def migrate(cr, version):
             ),
         )
 
-    # FIXME: leverage WEIRD_SEQUENCES
-
     logger.info("move pr data to batches...")
     cr.execute("""
     UPDATE runbot_merge_batch b
diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py
index 4114c944..3c645969 100644
--- a/runbot_merge/models/project_freeze/__init__.py
+++ b/runbot_merge/models/project_freeze/__init__.py
@@ -267,6 +267,11 @@ class FreezeWizard(models.Model):
                           bump.pr_id.display_name, prev, len(commits))
             bump_heads[repo_id] = repos[repo_id].rebase(prev, commits)[0]
 
+        # prevent concurrent updates to the commits table so we control the
+        # creation of commit objects from rebasing the release & bump PRs, do it
+        # only just before *pushing*
+        self.env.cr.execute("LOCK runbot_merge_commit IN ACCESS EXCLUSIVE MODE NOWAIT")
+
         deployed = {}
         # at this point we've got a bunch of tmp branches with merged release
         # and bump PRs, it's time to update the corresponding targets
@@ -350,6 +355,71 @@ class FreezeWizard(models.Model):
         )
         all_prs.batch_id.merge_date = fields.Datetime.now()
         all_prs.reviewed_by = self.env.user.partner_id.id
+        for p in all_prs:
+            p.commits_map = json.dumps({
+                '': deployed[p.id],
+                p.head: deployed[p.id]
+            })
+
+        # stagings have to be created conditionally as otherwise we might not
+        # have a `target` to set and it's mandatory
+        laster = self.env['runbot_merge.stagings'].search(
+            [('target', '=', master.id), ('state', '=', 'success')],
+            order='id desc',
+            limit=1,
+        ).commits.mapped(lambda c: (c.repository_id, c.commit_id))
+        if self.release_pr_ids:
+            rel_items = [(0, 0, {
+                    'repository_id': repo.id,
+                    'commit_id': self.env['runbot_merge.commit'].create({
+                        'sha': sha,
+                        'to_check': False,
+                    }).id,
+                } if (sha := rel_heads.get(repo)) else {
+                    'repository_id': repo.id,
+                    'commit_id': commit.id,
+                })
+                for repo, commit in laster
+            ]
+            self.env['runbot_merge.stagings'].create([{
+                'state': 'success',
+                'reason': 'release freeze staging',
+                'active': False,
+                'target': b.id,
+                'staging_batch_ids': [
+                    (0, 0, {'runbot_merge_batch_id': batch.id})
+                    for batch in self.release_pr_ids.pr_id.batch_id
+                ],
+                'heads': rel_items,
+                'commits': rel_items,
+            }])
+
+        if self.bump_pr_ids:
+            bump_items = [(0, 0, {
+                    'repository_id': repo.id,
+                    'commit_id': self.env['runbot_merge.commit'].create({
+                        'sha': sha,
+                        'to_check': False,
+                    }).id,
+                } if (sha := bump_heads.get(repo)) else {
+                    'repository_id': repo.id,
+                    'commit_id': commit.id,
+                })
+                for repo, commit in laster
+            ]
+            self.env['runbot_merge.stagings'].create([{
+                'state': 'success',
+                'reason': 'bump freeze staging',
+                'active': False,
+                'target': master.id,
+                'staging_batch_ids': [
+                    (0, 0, {'runbot_merge_batch_id': batch.id})
+                    for batch in self.bump_pr_ids.pr_id.batch_id
+                ],
+                'heads': bump_items,
+                'commits': bump_items,
+            }])
+
         self.env['runbot_merge.pull_requests.feedback'].create([{
             'repository': pr.repository.id,
             'pull_request': pr.number,
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 873986dc..2691a21e 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -1223,6 +1223,16 @@ def test_freeze_complete(env, project, repo_a, repo_b, repo_c, users, config):
     # stuff that's done directly
     assert all(pr_id.state == 'merged' for pr_id in release_pr_ids)
     assert pr_bump_id.state == 'merged'
+    assert pr_bump_id.commits_map != '{}'
+
+    assert len(release_pr_ids.batch_id) == 1
+    assert release_pr_ids.batch_id.merge_date
+    assert release_pr_ids.batch_id.staging_ids.target.name == '1.1'
+    assert release_pr_ids.batch_id.staging_ids.state == 'success'
+
+    assert pr_bump_id.batch_id.merge_date
+    assert pr_bump_id.batch_id.staging_ids.target.name == 'master'
+    assert pr_bump_id.batch_id.staging_ids.state == 'success'
 
     # stuff that's behind a cron
     env.run_crons()

From 232aa271b0f1b8da25ba48c9217837653de39150 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 5 Mar 2024 12:59:58 +0100
Subject: [PATCH 103/192] [ADD] runbot_merge: PR dashboard V2

Displays the entire batch set as a table, along both
repository (linked PRs) and branch (forward ports). Should provide a
much more complete overview.

Adds a copy of the dashboard as a raster render, to link from the PR:
as usual SVG is shit, content-based viewboxes are hell and having to
duplicate the entire CSS because `<img/>`-linked CSS can't run is
gross. And there's no payoff since the image is not interactible
anyway.

Performing manual ad-hoc table rendering via pillow is not
significantly worse, it works fine and it's possible to do *really*
good conditional request handling (hopefully) because I've basically
got all the information I need right here.

In fact it might make sense to upgrade the regular HTML page with
similar conditional request handling, at least for the last-update
bit if not the etag.

Fixes #771,fixes #770
---
 forwardport/data/views.xml                    |  25 --
 mergebot_test_utils/utils.py                  |   3 +-
 runbot_merge/__manifest__.py                  |   1 +
 runbot_merge/controllers/dashboard.py         | 235 +++++++++++++++++-
 ..._merge.pull_requests.feedback.template.csv |   2 +-
 runbot_merge/models/batch.py                  |   7 +-
 runbot_merge/static/scss/runbot_merge.scss    |  42 +++-
 runbot_merge/tests/test_basic.py              |   1 -
 runbot_merge/tests/test_disabled_branch.py    |   9 +-
 runbot_merge/tests/test_multirepo.py          |   6 +-
 runbot_merge/views/batch.xml                  |  59 +++++
 runbot_merge/views/mergebot.xml               |  51 ++--
 runbot_merge/views/templates.xml              | 167 ++++++++++++-
 13 files changed, 522 insertions(+), 86 deletions(-)
 create mode 100644 runbot_merge/views/batch.xml

diff --git a/forwardport/data/views.xml b/forwardport/data/views.xml
index 01af17e2..be8480d9 100644
--- a/forwardport/data/views.xml
+++ b/forwardport/data/views.xml
@@ -136,31 +136,6 @@
                              t-attf-title="@{{pr.reviewed_by.github_login}}"/>
                 </dd>
             </t>
-            <t t-if="pr.source_id">
-                <dt>forward-port of</dt>
-                <dd>
-                    <a t-att-href="pr.source_id.url">
-                        <span t-field="pr.source_id.display_name"/>
-                    </a>
-                    <span t-if="not pr.parent_id"
-                          class="badge badge-danger user-select-none"
-                          title="A detached PR behaves like a non-forward-port, it has to be approved via the mergebot, this is usually caused by the forward-port having been in conflict or updated.">
-                        DETACHED (<span t-out="pr.detach_reason" style="white-space: pre-wrap;"/>)
-                    </span>
-                </dd>
-            </t>
-            <t t-if="pr.forwardport_ids">
-                <dt>forward-ports</dt>
-                <dd><ul>
-                    <t t-foreach="pr.forwardport_ids" t-as="p">
-                        <t t-set="bgsignal"><t t-call="forwardport.pr_background"/></t>
-                        <li t-att-class="bgsignal">
-                            <a t-att-href="p.url"><span t-field="p.display_name"/></a>
-                            targeting <span t-field="p.target.name"/>
-                        </li>
-                    </t>
-                </ul></dd>
-            </t>
         </xpath>
     </template>
 
diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index b00bf520..c69b3a3c 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -56,7 +56,8 @@ class re_matches:
         return repr(str(self))
 
 def seen(env, pr, users):
-    return users['user'], f'[Pull request status dashboard]({to_pr(env, pr).url}).'
+    url = to_pr(env, pr).url
+    return users['user'], f'[![Pull request status dashboard]({url}.png)]({url})'
 
 def make_basic(
         env,
diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py
index 9799907d..13bed3dc 100644
--- a/runbot_merge/__manifest__.py
+++ b/runbot_merge/__manifest__.py
@@ -12,6 +12,7 @@
         'data/runbot_merge.pull_requests.feedback.template.csv',
         'views/res_partner.xml',
         'views/runbot_merge_project.xml',
+        'views/batch.xml',
         'views/mergebot.xml',
         'views/queues.xml',
         'views/configuration.xml',
diff --git a/runbot_merge/controllers/dashboard.py b/runbot_merge/controllers/dashboard.py
index b06ee8bf..0e19401c 100644
--- a/runbot_merge/controllers/dashboard.py
+++ b/runbot_merge/controllers/dashboard.py
@@ -1,13 +1,26 @@
 # -*- coding: utf-8 -*-
+from __future__ import annotations
+
+import base64
 import collections
+import colorsys
+import hashlib
+import io
 import json
+import math
 import pathlib
+from email.utils import formatdate
+from itertools import chain, product
+from typing import Tuple, cast, Mapping
 
 import markdown
 import markupsafe
 import werkzeug.exceptions
+import werkzeug.wrappers
+from PIL import Image, ImageDraw, ImageFont
 
 from odoo.http import Controller, route, request
+from odoo.tools import file_open
 
 LIMIT = 20
 class MergebotDashboard(Controller):
@@ -79,8 +92,8 @@ class MergebotDashboard(Controller):
             'entries': entries,
         })
 
-    @route('/<org>/<repo>/pull/<int(min=1):pr>', auth='public', type='http', website=True, sitemap=False)
-    def pr(self, org, repo, pr):
+    @route('/<org>/<repo>/pull/<int(min=1):pr><any("", ".png"):png>', auth='public', type='http', website=True, sitemap=False)
+    def pr(self, org, repo, pr, png):
         pr_id = request.env['runbot_merge.pull_requests'].sudo().search([
             ('repository.name', '=', f'{org}/{repo}'),
             ('number', '=', int(pr)),
@@ -90,6 +103,9 @@ class MergebotDashboard(Controller):
         if not pr_id.repository.group_id <= request.env.user.groups_id:
             raise werkzeug.exceptions.NotFound()
 
+        if png:
+            return raster_render(pr_id)
+
         st = {}
         if pr_id.statuses:
             # normalise `statuses` to map to a dict
@@ -102,3 +118,218 @@ class MergebotDashboard(Controller):
             'merged_head': json.loads(pr_id.commits_map).get(''),
             'statuses': st
         })
+
+def raster_render(pr):
+    default_headers = {
+        'Content-Type': 'image/png',
+        'Last-Modified': formatdate(),
+        # - anyone can cache the image, so public
+        # - crons run about every minute so that's how long a request is fresh
+        # - if the mergebot can't be contacted, allow using the stale response (no must-revalidate)
+        # - intermediate caches can recompress the PNG if they want (pillow is not a very good PNG generator)
+        # - the response is mutable even during freshness, technically (as there
+        #   is no guarantee the freshness window lines up with the cron, plus
+        #   some events are not cron-based)
+        # - maybe don't allow serving the stale image *while* revalidating?
+        # - allow serving a stale image for a day if the server returns 500
+        'Cache-Control': 'public, max-age=60, stale-if-error=86400',
+    }
+    if if_none_match := request.httprequest.headers.get('If-None-Match'):
+        # just copy the existing value out if we received any
+        default_headers['ETag'] = if_none_match
+
+    # weak validation: check the latest modification date of all objects involved
+    project, repos, branches, genealogy = pr.env.ref('runbot_merge.dashboard-pre')\
+        ._run_action_code_multi({'pr': pr})
+
+    # last-modified should be in RFC2822 format, which is what
+    # email.utils.formatdate does (sadly takes a timestamp but...)
+    last_modified = formatdate(max((
+        o.write_date
+        for o in chain(
+            project,
+            repos,
+            branches,
+            genealogy,
+            genealogy.all_prs | pr,
+        )
+    )).timestamp())
+    # The (304) response must not contain a body and must include the headers
+    # that would have been sent in an equivalent 200 OK response
+    headers = {**default_headers, 'Last-Modified': last_modified}
+    if request.httprequest.headers.get('If-Modified-Since') == last_modified:
+        return werkzeug.wrappers.Response(status=304, headers=headers)
+
+    with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Regular.ttf', 'rb') as f:
+        font = ImageFont.truetype(f, size=16, layout_engine=0)
+        f.seek(0)
+        supfont = ImageFont.truetype(f, size=10, layout_engine=0)
+    with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Bold.ttf', 'rb') as f:
+        bold = ImageFont.truetype(f, size=16, layout_engine=0)
+
+    batches = pr.env.ref('runbot_merge.dashboard-prep')._run_action_code_multi({
+        'pr': pr,
+        'repos': repos,
+        'branches': branches,
+        'genealogy': genealogy,
+    })
+
+    # getbbox returns (left, top, right, bottom)
+
+    rows = {b: font.getbbox(b.name)[3] for b in branches}
+    rows[None] = max(bold.getbbox(r.name)[3] for r in repos)
+
+    columns = {r: bold.getbbox(r.name)[2] for r in repos}
+    columns[None] = max(font.getbbox(b.name)[2] for b in branches)
+
+    etag = hashlib.sha256(f"(P){pr.id},{pr.repository.id},{pr.target.id}".encode())
+    # repos and branches should be in a consistent order so can just hash that
+    etag.update(''.join(f'(R){r.name}' for r in repos).encode())
+    etag.update(''.join(f'(T){b.name},{b.active}' for b in branches).encode())
+    # and product of deterministic iterations should be deterministic
+    for r, b in product(repos, branches):
+        ps = batches[r, b]
+        etag.update(f"(B){ps['state']},{ps['detached']},{ps['active']}".encode())
+        # technically label (state + blocked) does not actually impact image
+        # render (though subcomponents of state do) however blocked is useful
+        # to force an etag miss so keeping it
+        # TODO: blocked includes draft & merge method, maybe should change looks?
+        etag.update(''.join(
+            f"(PS){p['label']},{p['closed']},{p['number']},{p['checked']},{p['reviewed']},{p['attached']}"
+            for p in ps['prs']
+        ).encode())
+
+        w = h = 0
+        for p in ps['prs']:
+            _, _, ww, hh = font.getbbox(f" #{p['number']}")
+            w += ww + supfont.getbbox(' '.join(filter(None, [
+                'error' if p['pr'].error else '',
+                '' if p['checked'] else 'unchecked',
+                '' if p['reviewed'] else 'unreviewed',
+                '' if p['attached'] else 'detached',
+            ])))[2]
+            h = max(hh, h)
+        rows[b] = max(rows.get(b, 0), h)
+        columns[r] = max(columns.get(r, 0), w)
+
+    etag = headers['ETag'] = base64.b32encode(etag.digest()).decode()
+    if if_none_match == etag:
+        return werkzeug.wrappers.Response(status=304, headers=headers)
+
+    pad_w, pad_h = 20, 5
+    image_height = sum(rows.values()) + 2 * pad_h * len(rows)
+    image_width = sum(columns.values()) + 2 * pad_w * len(columns)
+    im = Image.new("RGB", (image_width+1, image_height+1), color='white')
+    draw = ImageDraw.Draw(im, 'RGB')
+    draw.font = font
+
+    # for reasons of that being more convenient we store the bottom of the
+    # current row, so getting the top edge requires subtracting h
+    w = left = bottom = 0
+    for b, r in product(chain([None], branches), chain([None], repos)):
+        left += w
+
+        opacity = 1.0 if b is None or b.active else 0.5
+        background = BG['info'] if b == pr.target or r == pr.repository else BG[None]
+        w, h = columns[r] + 2 * pad_w, rows[b] + 2 * pad_h
+
+        if r is None: # branch cell in row
+            left = 0
+            bottom += h
+            if b:
+                draw.rectangle(
+                    (left + 1, bottom - h + 1, left+w - 1, bottom - 1),
+                    background,
+                )
+                draw.text(
+                    (left + pad_w, bottom - h + pad_h),
+                    b.name,
+                    fill=blend(TEXT, opacity, over=background),
+                )
+        elif b is None: # repo cell in top row
+            draw.rectangle((left + 1, bottom - h + 1, left+w - 1, bottom - 1), background)
+            draw.text((left + pad_w, bottom - h + pad_h), r.name, fill=TEXT, font=bold)
+        # draw the bottom-right edges of the cell
+        draw.line([
+            (left, bottom), # bottom-left
+            (left + w, bottom), # bottom-right
+            (left+w, bottom-h) # top-right
+        ], fill=(172, 176, 170))
+        if r is None or b is None:
+            continue
+
+        ps = batches[r, b]
+
+        bgcolor = BG[ps['state']]
+        if pr in ps['pr_ids']:
+            bgcolor = lighten(bgcolor, by=-0.05)
+        background = blend(bgcolor, opacity, over=background)
+        draw.rectangle((left + 1, bottom - h + 1, left+w - 1, bottom - 1), background)
+
+        top = bottom - h + pad_h
+        offset = left + pad_w
+        for p in ps['prs']:
+            label = f"#{p['number']}"
+            foreground = blend((39, 110, 114), opacity, over=background)
+            draw.text((offset, top), label, fill=foreground)
+            x, _, ww, hh = font.getbbox(label)
+            if p['closed']:
+                draw.line([
+                    (offset+x, top + hh - hh/3),
+                    (offset+x+ww, top + hh - hh/3),
+                ], fill=foreground)
+            offset += ww
+            if not p['attached']:
+                # overdraw top border to mark the detachment
+                draw.line([(left, bottom-h), (left+w, bottom-h)], fill=ERROR)
+            for attribute in filter(None, [
+                'error' if p['pr'].error else '',
+                '' if p['checked'] else 'unchecked',
+                '' if p['reviewed'] else 'unreviewed',
+                '' if p['attached'] else 'detached',
+            ]):
+                label = f' {attribute}'
+                draw.text((offset, top), label,
+                          fill=blend(ERROR, opacity, over=background),
+                          font=supfont)
+                offset += supfont.getbbox(label)[2]
+            offset += math.ceil(supfont.getlength(" "))
+
+    buffer = io.BytesIO()
+    im.save(buffer, 'png', optimize=True)
+    return werkzeug.wrappers.Response(buffer.getvalue(), headers=headers)
+
+Color = Tuple[int, int, int]
+TEXT: Color = (102, 102, 102)
+ERROR: Color = (220, 53, 69)
+BG: Mapping[str | None, Color] = collections.defaultdict(lambda: (255, 255, 255), {
+    'info': (217, 237, 247),
+    'success': (223, 240, 216),
+    'warning': (252, 248, 227),
+    'danger': (242, 222, 222),
+})
+def blend_single(c: int, over: int, opacity: float) -> int:
+    return round(over * (1 - opacity) + c * opacity)
+
+def blend(color: Color, opacity: float, *, over: Color = (255, 255, 255)) -> Color:
+    assert 0.0 <= opacity <= 1.0
+    return (
+        blend_single(color[0], over[0], opacity),
+        blend_single(color[1], over[1], opacity),
+        blend_single(color[2], over[2], opacity),
+    )
+
+def lighten(color: Color, *, by: float) -> Color:
+    # colorsys uses values in the range [0, 1] rather than pillow/CSS-style [0, 225]
+    r, g, b = tuple(c / 255 for c in color)
+    hue, lightness, saturation = colorsys.rgb_to_hls(r, g, b)
+
+    # by% of the way between value and 1.0
+    if by >= 0: lightness += (1.0 - lightness) * by
+    # -by% of the way between 0 and value
+    else:lightness *= (1.0 + by)
+
+    return cast(Color, tuple(
+        round(c * 255)
+        for c in colorsys.hls_to_rgb(hue, lightness, saturation)
+    ))
diff --git a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
index d9e64d2f..5fe768ae 100644
--- a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
+++ b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
@@ -53,7 +53,7 @@ runbot_merge.failure.approved,{pr.ping}{status!r} failed on this reviewed PR.,"N
 
 pr: pull request in question
 status: failed status"
-runbot_merge.pr.created,[Pull request status dashboard]({pr.url}).,"Initial comment on PR creation.
+runbot_merge.pr.created,[![Pull request status dashboard]({pr.url}.png)]({pr.url}),"Initial comment on PR creation.
 
 pr: created pr"
 runbot_merge.pr.linked.not_ready,{pr.ping}linked pull request(s) {siblings} not ready. Linked PRs are not staged until all of them are ready.,"Comment when a PR is ready (approved & validated) but it is linked to other PRs which are not.
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index 1472ccd1..00886b1d 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -53,6 +53,7 @@ class Batch(models.Model):
     _inherit = ['mail.thread']
     _parent_store = True
 
+    name = fields.Char(compute="_compute_name")
     target = fields.Many2one('runbot_merge.branch', store=True, compute='_compute_target')
     batch_staging_ids = fields.One2many('runbot_merge.staging.batch', 'runbot_merge_batch_id')
     staging_ids = fields.Many2many(
@@ -176,6 +177,11 @@ class Batch(models.Model):
     def _search_open_prs(self, operator, value):
         return [('all_prs', operator, value), ('active', '=', True)]
 
+    @api.depends("prs.label")
+    def _compute_name(self):
+        for batch in self:
+            batch.name = batch.prs[:1].label or batch.all_prs[:1].label
+
     @api.depends("all_prs.target")
     def _compute_target(self):
         for batch in self:
@@ -190,7 +196,6 @@ class Batch(models.Model):
                 else:
                     batch.target = False
 
-
     @api.depends(
         "merge_date",
         "prs.error", "prs.draft", "prs.squash", "prs.merge_method",
diff --git a/runbot_merge/static/scss/runbot_merge.scss b/runbot_merge/static/scss/runbot_merge.scss
index 2e6577b3..e5a18687 100644
--- a/runbot_merge/static/scss/runbot_merge.scss
+++ b/runbot_merge/static/scss/runbot_merge.scss
@@ -14,27 +14,30 @@ h1, h2, h3, h4, h5, h6{
   margin-bottom: 0.33em;
 }
 h5 { font-size: 1em; }
-.bg-success, .bg-info, .bg-warning, .bg-danger, .bg-gray-lighter {
+.bg-success, .bg-info, .bg-warning, .bg-danger, .bg-gray-lighter,
+.table-success, .table-info, .table-warning, .table-danger {
   color: inherit;
 }
 .dropdown-item, .dropdown-menu, .dropdown-menu a {
   color: inherit;
 }
-.bg-success {
-  background-color: #dff0d8 !important;
+
+$mergebot-colors: ("success": #dff0d8, "danger": #f2dede, "warning": #fcf8e3, "info": #d9edf7);
+@each $category, $color in $mergebot-colors {
+  .bg-#{$category} {
+    background-color: $color !important;
+  }
+  .table-#{$category} {
+    background-color: $color !important;
+    &.table-active {
+      background-color: scale-color($color, $lightness: -5%) !important;
+    }
+  }
 }
 .bg-unmerged {
-  background-color: #dcefe8 !important
-}
-.bg-info {
-  background-color: #d9edf7 !important;
-}
-.bg-warning {
-  background-color: #fcf8e3 !important;
-}
-.bg-danger {
-  background-color: #f2dede !important;
+  background-color: #f8f0e3 !important
 }
+
 .list-inline {
   margin-bottom: 10px;
 }
@@ -121,3 +124,16 @@ dl.runbot-merge-fields {
   // works better for the left edge of the *box*
   @extend .border-left;
 }
+
+// batches sequence table in PR dashboard: mostly uses (customised) bootstrap
+// but some of the style is bespoke because inline styles don't work well with
+// CSP
+.closed {
+  text-decoration: line-through;
+}
+tr.inactive {
+  opacity: 0.5;
+}
+td.detached {
+  border-top: 2px solid map-get($theme-colors, "danger");
+}
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index e47ff180..3b616cb8 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -53,7 +53,6 @@ def test_trivial_flow(env, repo, page, users, config):
     )) == {
         'label': f"{config['github']['owner']}:other",
         'head': c1,
-        'target': 'master',
     }
 
     with repo:
diff --git a/runbot_merge/tests/test_disabled_branch.py b/runbot_merge/tests/test_disabled_branch.py
index 08cbf117..ed970974 100644
--- a/runbot_merge/tests/test_disabled_branch.py
+++ b/runbot_merge/tests/test_disabled_branch.py
@@ -59,12 +59,9 @@ def test_existing_pr_disabled_branch(env, project, make_repo, setreviewers, conf
     assert staging_id.reason == "Target branch deactivated by 'admin'."
 
     p = pr_page(page, pr)
-    target = dict(zip(
-        (e.text for e in p.cssselect('dl.runbot-merge-fields dt')),
-        (p.cssselect('dl.runbot-merge-fields dd'))
-    ))['target']
-    assert target.text_content() == 'other (inactive)'
-    assert target.get('class') == 'text-muted bg-warning'
+    [target] = p.cssselect('table tr.bg-info')
+    assert 'inactive' in target.classes
+    assert target[0].text_content() == "other"
 
     assert pr.comments == [
         (users['reviewer'], "hansen r+"),
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 2691a21e..42043f78 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -1109,11 +1109,9 @@ def test_multi_project(env, make_repo, setreviewers, users, config,
 
     assert pr1.comments == [
         (users['reviewer'], 'hansen r+'),
-        (users['user'], f'[Pull request status dashboard]({pr1_id.url}).'),
-    ]
-    assert pr2.comments == [
-        (users['user'], f'[Pull request status dashboard]({pr2_id.url}).'),
+        seen(env, pr1, users),
     ]
+    assert pr2.comments == [seen(env, pr2, users)]
 
 def test_freeze_complete(env, project, repo_a, repo_b, repo_c, users, config):
     """ Tests the freeze wizard feature (aside from the UI):
diff --git a/runbot_merge/views/batch.xml b/runbot_merge/views/batch.xml
new file mode 100644
index 00000000..89fc3902
--- /dev/null
+++ b/runbot_merge/views/batch.xml
@@ -0,0 +1,59 @@
+<odoo>
+    <record id="runbot_merge_batch_form" model="ir.ui.view">
+        <field name="name">Batch form</field>
+        <field name="model">runbot_merge.batch</field>
+        <field name="arch" type="xml">
+            <form>
+                <sheet>
+                    <div class="oe_title"><h1><field name="name"/></h1></div>
+                    <group>
+                        <group>
+                            <field name="target"/>
+                            <field name="merge_date"/>
+                            <field name="priority" attrs="{'invisible': [('merge_date', '!=', False)]}"/>
+                            <field name="skipchecks" widget="boolean_toggle" attrs="{'invisible': [('merge_date', '!=', False)]}"/>
+                            <field name="cancel_staging" widget="boolean_toggle" attrs="{'invisible': [('merge_date', '!=', False)]}"/>
+                            <field name="fw_policy"/>
+                        </group>
+                        <group>
+                            <field name="blocked"/>
+                        </group>
+                    </group>
+                    <group string="Pull Requests">
+                        <group colspan="4">
+                            <field name="all_prs" nolabel="1" readonly="1">
+                                <tree>
+                                    <field name="display_name"/>
+                                    <field name="repository"/>
+                                    <field name="state"/>
+                                </tree>
+                            </field>
+                        </group>
+                    </group>
+                    <group string="Genealogy">
+                        <group colspan="4">
+                            <field name="genealogy_ids" nolabel="1" readonly="1">
+                                <tree decoration-muted="id == parent.id">
+                                    <field name="name"/>
+                                    <field name="target"/>
+                                    <field name="all_prs" widget="many2many_tags"/>
+                                </tree>
+                            </field>
+                        </group>
+                    </group>
+                    <group string="Stagings">
+                        <group colspan="4">
+                            <field name="staging_ids" nolabel="1" readonly="1">
+                                <tree>
+                                    <field name="staged_at"/>
+                                    <field name="state"/>
+                                    <field name="reason"/>
+                                </tree>
+                            </field>
+                        </group>
+                    </group>
+                </sheet>
+            </form>
+        </field>
+    </record>
+</odoo>
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index 46e87ab8..88b33f4e 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -131,49 +131,41 @@
                         </h1>
                         <h2>
                           <field name="state"/>
-                          <span attrs="{'invisible': [('blocked', '=', False)]}">
-                            (<field name="blocked"/>)
+                          <span attrs="{'invisible': ['|', ('state', '=', 'merged'), ('blocked', '=', False)]}">
+                            (blocked: <field name="blocked"/>)
                           </span>
+                            <span attrs="{'invisible': [('state', '!=', 'merged')]}">
+                                (<field name="merge_date"/>)
+                            </span>
                         </h2>
                     </div>
-                    <!-- massively impactful status items -->
-                    <group>
-                      <group>
-                            <field name="skipchecks" widget="boolean_toggle"/>
-                      </group>
-                      <group>
-                            <field name="cancel_staging" widget="boolean_toggle"/>
-                      </group>
-                    </group>
                     <!-- main PR metadata -->
                     <group>
-                        <group name="metadata">
+                        <group name="metadata" colspan="2">
+                            <field name="batch_id"/>
                             <field name="target"/>
-                            <field name="label"/>
                             <field name="author"/>
                             <field name="head"/>
                         </group>
                         <group name="merging">
-                            <field name="priority"/>
                             <field name="merge_method"/>
                             <field name="squash"/>
                             <field name="draft"/>
-<!--                            <field string="Up To" name="limit_id"/>-->
-<!--                            <field string="Forward-Port Policy" name="fw_policy"/>-->
+                            <field name="priority"/>
+                            <field name="skipchecks" widget="boolean_toggle"/>
+                            <field name="cancel_staging" widget="boolean_toggle"/>
+                            <field name="limit_id"/>
                         </group>
                     </group>
                     <!-- influencers -->
                     <group string="State">
-                        <group>
-                            <field name="merge_date"/>
+                        <group colspan="4">
                             <field name="reviewed_by"/>
                             <field name="error"/>
                             <field name="closed"/>
-                        </group>
-                        <group name="status">
                             <field name="status"/>
-                            <field name="statuses"/>
                             <field name="overrides"/>
+                            <field name="statuses"/>
                         </group>
                     </group>
                     <group string="Forward Porting">
@@ -290,6 +282,15 @@
                             </field>
                         </group>
                     </group>
+                    <group string="Batches">
+                        <field name="batch_ids" colspan="4" nolabel="1" readonly="1">
+                            <tree>
+                                <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                <field name="name"/>
+                                <field name="prs" widget="many2many_tags"/>
+                            </tree>
+                        </field>
+                    </group>
                     <group string="PRs">
                         <field name="pr_ids" colspan="4" nolabel="1" readonly="1">
                             <tree>
@@ -300,14 +301,6 @@
                             </tree>
                         </field>
                     </group>
-                    <group string="Batches">
-                        <field name="batch_ids" colspan="4" nolabel="1" readonly="1">
-                            <tree>
-                                <button type="object" name="get_formview_action" icon="fa-external-link"/>
-                                <field name="prs" widget="many2many_tags"/>
-                            </tree>
-                        </field>
-                    </group>
                 </sheet>
             </form>
         </field>
diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index 7929208d..ef3cbe81 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -412,17 +412,178 @@
                     <t t-else="">open</t>
                 </t>
                 <t t-call="runbot_merge.view_pull_request_info_{{tmpl.strip()}}"/>
-                <t t-set="target_cls" t-value="None if pr.target.active else 'text-muted bg-warning'"/>
                 <dl class="runbot-merge-fields">
                     <dt>label</dt>
                     <dd><span t-field="pr.label"/></dd>
                     <dt>head</dt>
                     <dd><a t-attf-href="{{pr.github_url}}/commits/{{pr.head}}"><span t-field="pr.head"/></a></dd>
-                    <dt t-att-class="target_cls">target</dt>
-                    <dd t-att-class="target_cls"><span t-field="pr.target"/></dd>
                 </dl>
+                <t t-call="runbot_merge.dashboard-table"/>
                 <p t-field="pr.message"/>
             </div></div>
         </t>
     </template>
+
+    <record id="dashboard-pre" model="ir.actions.server">
+        <field name="name">Preparation for the preparation of the PR dashboard content</field>
+        <field name="state">code</field>
+        <field name="model_id" ref="base.model_ir_qweb"/>
+        <field name="code"><![CDATA[
+project = pr.repository.project_id
+genealogy = pr.batch_id.genealogy_ids
+repos = project.repo_ids & genealogy.all_prs.repository
+targets = genealogy.all_prs.target
+if not genealogy:
+    # if a PR is closed, it may not have a batch to get a genealogy from,
+    # in which case it's just a sole soul drifting in the deep dark
+    branches = pr.target
+    repos = pr.repository
+elif all(p.state in ('merged', 'closed') for p in genealogy[-1].all_prs):
+    branches = (project.branch_ids & targets)[::-1]
+else:
+    # if the tip of the genealogy is not closed, extend to the furthest limit,
+    # keeping branches which are active or have an associated batch / PR
+    limit = genealogy.prs.limit_id.sorted(lambda b: (b.sequence, b.name))
+    limit_high = project.branch_ids.ids.index(limit.id) if limit else None
+    limit = targets.sorted(lambda b: (b.sequence, b.name))[-1]
+    limit_low = project.branch_ids.ids.index(limit.id)
+    branches = project.branch_ids[limit_high:limit_low+1].filtered(lambda b: b.active or b in targets)[::-1]
+
+action = (project, repos, branches, genealogy)
+        ]]></field>
+    </record>
+
+    <record id="dashboard-prep" model="ir.actions.server">
+        <field name="name">Preparation of the PR dashboard content</field>
+        <field name="state">code</field>
+        <field name="model_id" ref="base.model_ir_qweb"/>
+        <field name="code"><![CDATA[
+batches = {}
+for branch in branches:
+    # FIXME: batches with inconsistent targets?
+    if genealogy:
+        prs_batch = genealogy.filtered(lambda b: b.target == branch).all_prs
+    else:
+        prs_batch = pr
+    for repo in repos:
+        prs = prs_batch.filtered(lambda p: p.repository == repo)
+        st = 0
+        detached = False
+        pr_fmt = []
+        for p in prs:
+            st |= (bool(p.error) << 2 | (p.state == 'merged') << 1 | bool(p.blocked) << 0)
+
+            done = p.state in ('closed', 'merged')
+            # this will hide the detachment signal when the PRs are merged/closed, cleaner but less correct?
+            detached = detached or bool(p.source_id and not p.parent_id and not done)
+            label = p.state
+            if p.blocked:
+                label = "%s, %s" % (label, p.blocked)
+            pr_fmt.append({
+                'pr': p,
+                'number': p.number,
+                'label': label,
+                'closed': p.closed,
+                'backend_url': "/web#view_type=form&model=runbot_merge.pull_requests&id=%d" % p.id,
+                'github_url': p.github_url,
+                'checked': done or p.status == 'success',
+                'reviewed': done or bool(p.reviewed_by),
+                'attached': done or p.parent_id or not p.source_id,
+            })
+        state = None
+        for i, s in zip(range(2, -1, -1), ['danger', 'success', 'warning']):
+            if st & (1 << i):
+                state = s
+                break
+
+        batches[repo, branch] = {
+            'active': pr in prs,
+            'detached': detached,
+            'state': state,
+            'prs': pr_fmt,
+            'pr_ids': prs,
+        }
+
+action = batches
+        ]]></field>
+    </record>
+    <template id="dashboard-table">
+        <t t-set="pre" t-value="pr.env.ref('runbot_merge.dashboard-pre').sudo()._run_action_code_multi({'pr': pr})"/>
+        <t t-set="repos" t-value="pre[1]"/>
+        <t t-set="branches" t-value="pre[2]"/>
+        <t t-set="batches" t-value="env.ref('runbot_merge.dashboard-prep').sudo()._run_action_code_multi({
+            'pr': pr,
+            'repos': repos,
+            'branches': branches,
+            'genealogy': pre[3],
+        })"/>
+        <table class="table table-bordered table-sm">
+            <colgroup>
+                <col/>
+                <col t-foreach="repos" t-as="repo"
+                     t-att-class="'bg-info' if repo == pr.repository else None"
+                />
+            </colgroup>
+            <thead>
+                <tr>
+                    <th/>
+                    <th t-foreach="repos" t-as="repo">
+                        <t t-out="repo.name"/>
+                    </th>
+                </tr>
+            </thead>
+            <tbody>
+                <!--
+                table-info looks like shit (possibly because no odoo styling so use bg-info
+                text-muted doesn't do anything, so set some opacity
+                -->
+                <tr t-foreach="branches" t-as="branch"
+                    t-att-title="None if branch.active else 'branch is disabled'"
+                    t-attf-class="{{
+                        'bg-info' if branch == pr.target else ''
+                    }} {{
+                        'inactive' if not branch.active else ''
+                    }}">
+                    <td t-out="branch.name"/>
+                    <t t-foreach="repos" t-as="repo">
+                        <t t-set="ps" t-value="batches[repo, branch]"/>
+                        <t t-set="stateclass" t-value="ps['state'] and 'table-'+ps['state']"/>
+                        <t t-set="detached" t-value="ps['detached']"/>
+                        <td t-if="ps['prs']"
+                            t-att-title="'detached' if detached else None"
+                            t-attf-class="{{
+                                'table-active' if ps['active'] else ''
+                            }} {{
+                                'detached' if detached else ''
+                            }}{{stateclass}}">
+                            <!--
+                            there should be only one PR per (repo, target) but
+                            that's not always the case
+                            -->
+                            <span t-foreach="ps['prs']" t-as="p"
+                                  t-att-title="p['label']"
+                                  t-att-class="'closed' if p['closed'] else None">
+                                <a t-attf-href="/{{repo.name}}/pull/{{p['number']}}">#<t t-out="p['number']"/></a>
+                                <a t-attf-class="fa fa-brands fa-github"
+                                   title="Open on Github"
+                                   t-att-href="p['github_url']"
+                                />
+                                <a groups="base.group_user"
+                                   title="Open in Backend"
+                                   t-attf-class="fa fa-external-link"
+                                   t-att-href="p['backend_url']"
+                                />
+                                <sup t-if="not p['checked']" class="text-danger">unchecked</sup>
+                                <sup t-if="not p['reviewed']" class="text-danger">unreviewed</sup>
+                                <sup t-if="not p['attached']"
+                                     t-attf-title="detached: {{p['pr'].detach_reason}}"
+                                     class="text-warning fa fa-unlink"/>
+                            </span>
+                        </td>
+                        <td t-else=""/>
+                    </t>
+                </tr>
+            </tbody>
+        </table>
+    </template>
 </odoo>

From 3c3100adfe8e8269936bf5589ac57c059766e1bc Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 25 Apr 2024 12:04:26 +0200
Subject: [PATCH 104/192] [IMP] runbot_merge: cleanup PR backend

Shove a bunch of stuff in notebook tabs, add a few
affordances (e.g. github and frontend links, links from m2m), surface
a few missing fields.

Hopefully makes the backend form both easier to navigate and easier to
administrate from.
---
 runbot_merge/models/pull_requests.py |   6 ++
 runbot_merge/views/batch.xml         |   3 +
 runbot_merge/views/mergebot.xml      | 149 ++++++++++++++++++---------
 3 files changed, 108 insertions(+), 50 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 3e706268..90da29f6 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -393,12 +393,18 @@ class PullRequests(models.Model):
 
     batch_id = fields.Many2one('runbot_merge.batch', index=True)
     staging_id = fields.Many2one('runbot_merge.stagings', compute='_compute_staging', store=True)
+    staging_ids = fields.Many2many('runbot_merge.stagings', string="Stagings", compute='_compute_stagings', context={"active_test": False})
 
     @api.depends('batch_id.staging_ids.active')
     def _compute_staging(self):
         for p in self:
             p.staging_id = p.batch_id.staging_ids.filtered('active')
 
+    @api.depends('batch_id.staging_ids')
+    def _compute_stagings(self):
+        for p in self:
+            p.staging_ids = p.batch_id.staging_ids
+
     commits_map = fields.Char(help="JSON-encoded mapping of PR commits to actually integrated commits. The integration head (either a merge commit or the PR's topmost) is mapped from the 'empty' pr commit (the key is an empty string, because you can't put a null key in json maps).", default='{}')
 
     link_warned = fields.Boolean(
diff --git a/runbot_merge/views/batch.xml b/runbot_merge/views/batch.xml
index 89fc3902..7d87d94f 100644
--- a/runbot_merge/views/batch.xml
+++ b/runbot_merge/views/batch.xml
@@ -23,6 +23,7 @@
                         <group colspan="4">
                             <field name="all_prs" nolabel="1" readonly="1">
                                 <tree>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
                                     <field name="display_name"/>
                                     <field name="repository"/>
                                     <field name="state"/>
@@ -34,6 +35,7 @@
                         <group colspan="4">
                             <field name="genealogy_ids" nolabel="1" readonly="1">
                                 <tree decoration-muted="id == parent.id">
+                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
                                     <field name="name"/>
                                     <field name="target"/>
                                     <field name="all_prs" widget="many2many_tags"/>
@@ -45,6 +47,7 @@
                         <group colspan="4">
                             <field name="staging_ids" nolabel="1" readonly="1">
                                 <tree>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
                                     <field name="staged_at"/>
                                     <field name="state"/>
                                     <field name="reason"/>
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index 88b33f4e..c1cea06b 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -123,7 +123,11 @@
         <field name="model">runbot_merge.pull_requests</field>
         <field name="arch" type="xml">
             <form>
-                <header/>
+                <div class="o_form_statusbar">
+                    <span class="o_statusbar_buttons">
+                    <field name="github_url" widget="url" class="btn btn-secondary" text="Github"/>
+                    <field name="url" widget="url" class="btn btn-secondary" text="Frontend"/></span>
+                </div>
                 <sheet>
                     <div class="oe_title">
                         <h1>
@@ -140,67 +144,112 @@
                         </h2>
                     </div>
                     <!-- main PR metadata -->
-                    <group>
-                        <group name="metadata" colspan="2">
+                    <group name="metadata">
+                        <group>
                             <field name="batch_id"/>
                             <field name="target"/>
+                        </group>
+                        <group>
                             <field name="author"/>
                             <field name="head"/>
                         </group>
-                        <group name="merging">
-                            <field name="merge_method"/>
-                            <field name="squash"/>
-                            <field name="draft"/>
-                            <field name="priority"/>
-                            <field name="skipchecks" widget="boolean_toggle"/>
-                            <field name="cancel_staging" widget="boolean_toggle"/>
-                            <field name="limit_id"/>
-                        </group>
                     </group>
+                    <notebook>
+                        <page name="state" string="State">
+                            <group>
+                                <group>
+                                    <field name="reviewed_by"/>
+                                    <field name="closed"/>
+                                    <field name="error"/>
+                                </group>
+                                <group>
+                                    <field name="status"/>
+                                    <details colspan="4">
+                                        <summary>Commit Statuses</summary>
+
+                                        <field name="statuses"/>
+                                    </details>
+                                    <details colspan="4">
+                                        <summary>Overrides</summary>
+                                        <field name="overrides"/>
+                                    </details>
+                                </group>
+                            </group>
+                            <group>
+                                <group colspan="4">
+                                    <field name="blocked"/>
+                                </group>
+                            </group>
+                        </page>
+                        <page name="configuration" string="Configuration">
+                            <group>
+                                <group>
+                                    <field name="merge_method"/>
+                                    <field name="squash"/>
+                                    <field name="draft"/>
+                                    <field name="limit_id"/>
+                                </group>
+                                <group>
+                                    <field name="priority"/>
+                                    <field name="skipchecks" widget="boolean_toggle"/>
+                                    <field name="cancel_staging" widget="boolean_toggle"/>
+                                </group>
+                            </group>
+                            <group string="Delegates">
+                                <group colspan="4">
+                                    <field name="delegates" nolabel="1">
+                                        <tree>
+                                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                            <field name="name"/>
+                                            <field name="github_login"/>
+                                        </tree>
+                                    </field>
+                                </group>
+                            </group>
+                        </page>
+                        <page name="stagings" string="Staging History">
+                            <group>
+                                <group colspan="4">
+                                    <field name="staging_ids" nolabel="1" readonly="1">
+                                        <tree>
+                                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                            <field name="staged_at"/>
+                                            <field name="state"/>
+                                            <field name="reason"/>
+                                        </tree>
+                                    </field>
+                                </group>
+                            </group>
+                        </page>
+                        <page name="porting" string="Forward-Porting">
+                            <group>
+                                <group>
+                                    <field string="Original PR" name="source_id"/>
+                                    <field name="parent_id"/>
+                                    <field
+                                        attrs="{'invisible': [('parent_id', '=', False), ('source_id', '!=', False)]}"
+                                        string="Detached because" name="detach_reason" readonly="1"/>
+                                </group>
+                            </group>
+                            <group>
+                              <group colspan="4">
+                                <field name="forwardport_ids" nolabel="1" readonly="True">
+                                  <tree>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                    <field name="target" string="Branch"/>
+                                    <field name="number"/>
+                                  </tree>
+                                </field>
+                              </group>
+                            </group>
+                        </page>
+                    </notebook>
                     <!-- influencers -->
-                    <group string="State">
-                        <group colspan="4">
-                            <field name="reviewed_by"/>
-                            <field name="error"/>
-                            <field name="closed"/>
-                            <field name="status"/>
-                            <field name="overrides"/>
-                            <field name="statuses"/>
-                        </group>
-                    </group>
-                    <group string="Forward Porting">
-                      <group>
-                          <field string="Original PR" name="source_id"/>
-                          <field name="parent_id"/>
-                          <field
-                              attrs="{'invisible': [('parent_id', '=', False), ('source_id', '!=', False)]}"
-                              string="Detached because" name="detach_reason" readonly="1"/>
-                      </group>
-                      <group>
-                        <field name="forwardport_ids" nolabel="1" readonly="True">
-                          <tree>
-                            <field name="target" string="Branch"/>
-                            <field name="number"/>
-                          </tree>
-                        </field>
-                      </group>
-                    </group>
                     <group string="Message">
                         <group colspan="4">
                             <field name="message" nolabel="1"/>
                         </group>
                     </group>
-                    <group string="Delegates">
-                        <group colspan="4">
-                            <field name="delegates" nolabel="1">
-                                <tree>
-                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
-                                    <field name="name"/>
-                                    <field name="github_login"/>
-                                </tree>
-                            </field>
-                        </group>
-                    </group>
                 </sheet>
                 <div class="oe_chatter">
                     <field name="message_follower_ids" widget="mail_followers"/>

From 3f4519d60549c88c303ba80a5e3a328e96103e0f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 30 May 2024 10:55:40 +0200
Subject: [PATCH 105/192] [CHG] runbot_merge: add signoff & related to all
 commits

if rebased. Untouched commits (straight merge) remain unalterated, but
all rebased or squashed commits now get signoff and `Related` headers
added on top of the already previously added `part-of`.

Implement by generalising `_build_merge_message` to `_build_message`
and having `add_self_references` delegate to it, removes some of the
redundancy / differential handling.

Update the `part_of` helper to also add the S-O-B header to the PR,
although it currently does not reference the entire forward port
chain.

Fixes #876
---
 forwardport/models/project.py          | 22 -----------------
 mergebot_test_utils/utils.py           |  5 +++-
 runbot_merge/models/pull_requests.py   | 25 ++++++++++++++++---
 runbot_merge/models/stagings_create.py | 34 +++++++++++++-------------
 runbot_merge/tests/test_basic.py       |  3 ++-
 5 files changed, 44 insertions(+), 45 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 43379eb1..a4099d05 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -519,28 +519,6 @@ stderr:
             prev = configured.stdout().rev_parse('HEAD').stdout.decode()
             logger.info('%s: success -> %s', commit_sha, prev)
 
-    def _build_merge_message(self, message, related_prs=()):
-        msg = super()._build_merge_message(message, related_prs=related_prs)
-
-        # ensures all reviewers in the review path are on the PR in order:
-        # original reviewer, then last conflict reviewer, then current PR
-        reviewers = (self | self.root_id | self.source_id)\
-            .mapped('reviewed_by.formatted_email')
-
-        sobs = msg.headers.getlist('signed-off-by')
-        msg.headers.remove('signed-off-by')
-        msg.headers.extend(
-            ('signed-off-by', signer)
-            for signer in sobs
-            if signer not in reviewers
-        )
-        msg.headers.extend(
-            ('signed-off-by', reviewer)
-            for reviewer in reversed(reviewers)
-        )
-
-        return msg
-
     def _make_fp_message(self, commit):
         cmap = json.loads(self.commits_map)
         msg = Message.from_message(commit['commit']['message'])
diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index c69b3a3c..cba96114 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -182,7 +182,10 @@ def to_pr(env, pr):
 def part_of(label, pr_id, *, separator='\n\n'):
     """ Adds the "part-of" pseudo-header in the footer.
     """
-    return f'{label}{separator}Part-of: {pr_id.display_name}'
+    return f"""\
+{label}{separator}\
+Part-of: {pr_id.display_name}
+Signed-off-by: {pr_id.reviewed_by.formatted_email}"""
 
 def ensure_one(records):
     assert len(records) == 1
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 90da29f6..1a174aef 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1339,19 +1339,36 @@ class PullRequests(models.Model):
             if commit:
                 self.env.cr.commit()
 
-    def _build_merge_message(self, message: Union['PullRequests', str], related_prs=()) -> 'Message':
+    def _build_message(self, message: Union['PullRequests', str], related_prs: 'PullRequests' = (), merge: bool = True) -> 'Message':
         # handle co-authored commits (https://help.github.com/articles/creating-a-commit-with-multiple-authors/)
         m = Message.from_message(message)
         if not is_mentioned(message, self):
-            m.body += f'\n\ncloses {self.display_name}'
+            if merge:
+                m.body += f'\n\ncloses {self.display_name}'
+            else:
+                m.headers.pop('Part-Of', None)
+                m.headers.add('Part-Of', self.display_name)
 
         for r in related_prs:
             if not is_mentioned(message, r, full_reference=True):
                 m.headers.add('Related', r.display_name)
 
-        if self.reviewed_by:
-            m.headers.add('signed-off-by', self.reviewed_by.formatted_email)
+        # ensures all reviewers in the review path are on the PR in order:
+        # original reviewer, then last conflict reviewer, then current PR
+        reviewers = (self | self.root_id | self.source_id)\
+            .mapped('reviewed_by.formatted_email')
 
+        sobs = m.headers.getlist('signed-off-by')
+        m.headers.remove('signed-off-by')
+        m.headers.extend(
+            ('signed-off-by', signer)
+            for signer in sobs
+            if signer not in reviewers
+        )
+        m.headers.extend(
+            ('signed-off-by', reviewer)
+            for reviewer in reversed(reviewers)
+        )
         return m
 
     def unstage(self, reason, *args):
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index 1f620154..ec3d99b3 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -454,7 +454,7 @@ def stage(pr: PullRequests, info: StagingSlice, related_prs: PullRequests) -> Tu
     return method, new_head
 
 def stage_squash(pr: PullRequests, info: StagingSlice, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
-    msg = pr._build_merge_message(pr, related_prs=related_prs)
+    msg = pr._build_message(pr, related_prs=related_prs)
 
     authors = {
         (c['commit']['author']['name'], c['commit']['author']['email'])
@@ -499,11 +499,7 @@ def stage_squash(pr: PullRequests, info: StagingSlice, commits: List[github.PrCo
     return head
 
 def stage_rebase_ff(pr: PullRequests, info: StagingSlice, commits: List[github.PrCommit], related_prs: PullRequests) -> str:
-    # updates head commit with PR number (if necessary) then rebases
-    # on top of target
-    msg = pr._build_merge_message(commits[-1]['commit']['message'], related_prs=related_prs)
-    commits[-1]['commit']['message'] = str(msg)
-    add_self_references(pr, commits[:-1])
+    add_self_references(pr, commits, related_prs=related_prs, merge=commits[-1])
 
     _logger.debug("rebasing %s on %s (commits=%s)",
                   pr.display_name, info.head, len(commits))
@@ -512,11 +508,11 @@ def stage_rebase_ff(pr: PullRequests, info: StagingSlice, commits: List[github.P
     return head
 
 def stage_rebase_merge(pr: PullRequests, info: StagingSlice, commits: List[github.PrCommit], related_prs: PullRequests) -> str :
-    add_self_references(pr, commits)
+    add_self_references(pr, commits, related_prs=related_prs)
     _logger.debug("rebasing %s on %s (commits=%s)",
                   pr.display_name, info.head, len(commits))
     h, mapping = info.repo.rebase(info.head, commits=commits)
-    msg = pr._build_merge_message(pr, related_prs=related_prs)
+    msg = pr._build_message(pr, related_prs=related_prs)
 
     project = pr.repository.project_id
     merge_head= info.repo.merge(
@@ -554,7 +550,7 @@ def stage_merge(pr: PullRequests, info: StagingSlice, commits: List[github.PrCom
             raise exceptions.MergeError(pr, t.stderr)
         merge_tree = t.stdout.strip()
         new_parents = [info.head] + list(head_parents - {base_commit})
-        msg = pr._build_merge_message(pr_head['commit']['message'], related_prs=related_prs)
+        msg = pr._build_message(pr_head['commit']['message'], related_prs=related_prs)
 
         d2t = itemgetter('name', 'email', 'date')
         c = info.repo.commit_tree(
@@ -574,7 +570,7 @@ def stage_merge(pr: PullRequests, info: StagingSlice, commits: List[github.PrCom
         return copy
     else:
         # otherwise do a regular merge
-        msg = pr._build_merge_message(pr)
+        msg = pr._build_message(pr)
         project = pr.repository.project_id
         merge_head = info.repo.merge(
             info.head, pr.head, str(msg),
@@ -595,17 +591,21 @@ def is_mentioned(message: Union[PullRequests, str], pr: PullRequests, *, full_re
         pattern = fr'( |\b{repository})#{pr.number}\b'
     return bool(re.search(pattern, message if isinstance(message, str) else message.message))
 
-def add_self_references(pr: PullRequests, commits: List[github.PrCommit]):
+def add_self_references(
+        pr: PullRequests,
+        commits: List[github.PrCommit],
+        related_prs: PullRequests,
+        merge: Optional[github.PrCommit] = None,
+):
     """Adds a footer reference to ``self`` to all ``commits`` if they don't
     already refer to the PR.
     """
     for c in (c['commit'] for c in commits):
-        if not is_mentioned(c['message'], pr):
-            message = c['message']
-            m = Message.from_message(message)
-            m.headers.pop('Part-Of', None)
-            m.headers.add('Part-Of', pr.display_name)
-            c['message'] = str(m)
+        c['message'] = str(pr._build_message(
+            c['message'],
+            related_prs=related_prs,
+            merge=merge and c['url'] == merge['commit']['url'],
+        ))
 
 BREAK = re.compile(r'''
     [ ]{0,3} # 0-3 spaces of indentation
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 3b616cb8..afa7a3a9 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -1965,7 +1965,8 @@ Some: thing
 
 is odd
 
-Part-of: {pr_id.display_name}"""
+Part-of: {pr_id.display_name}
+Signed-off-by: {reviewer}"""
 
     def test_pr_mergehead(self, repo, env, config):
         """ if the head of the PR is a merge commit and one of the parents is

From 67f1c1e28818df6e1d8cd041edd0cf85de482992 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 30 May 2024 15:11:38 +0200
Subject: [PATCH 106/192] [IMP] runbot_merge: add staging duration

Computed on the fly for now. Formatted nicely in the frontend, there
does not seem to be any sort of duration widget in the backend so
just display the integer number of seconds.

Fixes #865
---
 runbot_merge/__manifest__.py                       |  2 +-
 runbot_merge/migrations/15.0.1.13/pre-migration.py |  4 ++++
 runbot_merge/models/pull_requests.py               | 10 +++++++++-
 runbot_merge/views/mergebot.xml                    |  2 ++
 runbot_merge/views/templates.xml                   | 12 +++++++++++-
 5 files changed, 27 insertions(+), 3 deletions(-)
 create mode 100644 runbot_merge/migrations/15.0.1.13/pre-migration.py

diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py
index 13bed3dc..b005c5a9 100644
--- a/runbot_merge/__manifest__.py
+++ b/runbot_merge/__manifest__.py
@@ -1,6 +1,6 @@
 {
     'name': 'merge bot',
-    'version': '1.12',
+    'version': '1.13',
     'depends': ['contacts', 'mail', 'website'],
     'data': [
         'security/security.xml',
diff --git a/runbot_merge/migrations/15.0.1.13/pre-migration.py b/runbot_merge/migrations/15.0.1.13/pre-migration.py
new file mode 100644
index 00000000..ed9ba615
--- /dev/null
+++ b/runbot_merge/migrations/15.0.1.13/pre-migration.py
@@ -0,0 +1,4 @@
+def migrate(cr, version):
+    cr.execute("ALTER TABLE runbot_merge_stagings "
+               "ADD COLUMN staging_end timestamp without time zone")
+    cr.execute("UPDATE runbot_merge_stagings SET staging_end = write_date")
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 1a174aef..f5cf0235 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1796,6 +1796,8 @@ class Stagings(models.Model):
     active = fields.Boolean(default=True)
 
     staged_at = fields.Datetime(default=fields.Datetime.now, index=True)
+    staging_end = fields.Datetime()
+    staging_duration = fields.Float(compute='_compute_duration')
     timeout_limit = fields.Datetime(store=True, compute='_compute_timeout_limit')
     reason = fields.Text("Reason for final state (if any)")
 
@@ -1819,11 +1821,17 @@ class Stagings(models.Model):
         for staging in previously_pending:
             if staging.state != 'pending':
                 super(Stagings, staging).write({
-                    'statuses_cache': json.dumps(staging.statuses)
+                    'staging_end': fields.Datetime.now(),
+                    'statuses_cache': json.dumps(staging.statuses),
                 })
 
         return True
 
+    @api.depends('staged_at', 'staging_end')
+    def _compute_duration(self):
+        for s in self:
+            s.staging_duration = ((s.staging_end or fields.Datetime.now()) - s.staged_at).total_seconds()
+
     def name_get(self):
         return [
             (staging.id, "%d (%s, %s%s)" % (
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index c1cea06b..c1d7418a 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -311,6 +311,8 @@
                         </group>
                         <group>
                             <field name="staged_at"/>
+                            <field string="Staging Duration (seconds)"
+                                    name="staging_duration" widget="integer"/>
                         </group>
                     </group>
                     <group>
diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index ef3cbe81..3b8d4d14 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -24,7 +24,7 @@
                     data-toggle="dropdown"
                     aria-haspopup="true"
                     aria-expanded="true"
-                    t-attf-title="Staged at {{staging.staged_at}}Z"
+                    t-attf-title="Staged at {{staging.staged_at}}Z for {{round(staging.staging_duration)}}s"
             >
                 <t t-out="0"/>
                 <span class="caret"></span>
@@ -187,6 +187,11 @@
                     </ul>
                     <t t-call="runbot_merge.staging-statuses">
                         Staged <span t-field="staging.staged_at" t-options="{'widget': 'relative'}"/>
+                        (duration <span t-field="staging.staging_duration" t-options="{
+                            'widget': 'duration',
+                            'format': 'short',
+                            'round': 'minute'
+                        }"/>)
                     </t>
                 </li>
             </t>
@@ -248,6 +253,11 @@
                                 <t t-call="runbot_merge.staging-statuses">
                                     <span t-field="staging.staged_at"
                                           t-options="{'format': 'yyyy-MM-dd\'T\'HH:mm:ssZ'}"/>
+                                    in <span t-field="staging.staging_duration" t-options="{
+                                        'widget': 'duration',
+                                        'format': 'narrow',
+                                        'round': 'minute'
+                                    }"/>
                                 </t>
                             </th>
                             <td>

From 68cfeddaed3eede755b3938cdb5042442b3cc4a7 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 30 May 2024 15:28:25 +0200
Subject: [PATCH 107/192] [ADD] runbot_merge: display required statuses after
 merge

Github makes it painfully difficult to access the statuses (especially
their URL / related build) once a PR has been merged, as it's
necessary to find the last non-staging commit mention / update in
order to find its statuses checkbox thingie, open that, and access the
statuses.

The mergebot has all the links, so it can just display them in the
merged mode as well rather than only display them in open mode. That
way even on a merged PR the statuses are just two clicks away.

Fixes #873
---
 runbot_merge/views/templates.xml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index 3b8d4d14..6a9c6060 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -321,6 +321,15 @@
             <t t-if="merged_head">
                 at <a t-attf-href="https://github.com/{{pr.repository.name}}/commit/{{merged_head}}"><t t-esc="merged_head"/></a>
             </t>
+            <p>Statuses:</p>
+            <ul>
+                <t t-foreach="pr.repository.status_ids._for_pr(pr)" t-as="ci">
+                    <t t-set="st" t-value="statuses.get(ci.context.strip())"/>
+                    <li t-if="st">
+                        <a t-att-href="st.get('target_url') if st else None"><t t-esc="ci.context.strip()"/></a><t t-if="st and st.get('description')">: <t t-esc="st['description']"/></t>
+                    </li>
+                </t>
+            </ul>
 
             <t t-set="linked_prs" t-value="pr._linked_prs"/>
             <div t-if="linked_prs">

From 44084e303ccece3cb54128ab29eab399bd4d24e9 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 31 May 2024 12:33:13 +0200
Subject: [PATCH 108/192] [REF] runbot_merge: compute staging state

Rather than compute staging state directly from commit statuses, copy
statuses into the staging's `statuses_cache` then compute state based
on that. Also refactor `statuses` and `staging_end` to be computed
based on the statuses cache instead of the commits (or `state`
update).

The goal is to allow non-webhook validation of stagings, for direct
communications between the mergebot and the CI (mostly runbot).
---
 runbot_merge/models/project.py       |  1 +
 runbot_merge/models/pull_requests.py | 65 ++++++++++++++++------------
 2 files changed, 39 insertions(+), 27 deletions(-)

diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index b3b55a8c..dee87735 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -32,6 +32,7 @@ class Project(models.Model):
         ('largest', "Largest of split and ready PRs"),
         ('ready', "Ready PRs over split"),
     ], default="default", required=True)
+    staging_statuses = fields.Boolean(default=True)
 
     ci_timeout = fields.Integer(
         default=60, required=True, group_operator=None,
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index f5cf0235..a8716ba7 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1740,9 +1740,13 @@ class Commit(models.Model):
                         f"statuses changed on {c.sha}"
                     pr._validate(c.statuses)
 
-                stagings = Stagings.search([('head_ids.sha', '=', c.sha), ('state', '=', 'pending')])
+                stagings = Stagings.search([
+                    ('head_ids.sha', '=', c.sha),
+                    ('state', '=', 'pending'),
+                    ('target.project_id.staging_statuses', '=', True),
+                ])
                 if stagings:
-                    stagings._validate()
+                    stagings._notify(c)
             except Exception:
                 _logger.exception("Failed to apply commit %s (%s)", c, c.sha)
                 self.env.cr.rollback()
@@ -1792,11 +1796,11 @@ class Stagings(models.Model):
         ('pending', 'Pending'),
         ('cancelled', "Cancelled"),
         ('ff_failed', "Fast forward failed")
-    ], default='pending', index=True)
+    ], default='pending', index=True, store=True, compute='_compute_state')
     active = fields.Boolean(default=True)
 
     staged_at = fields.Datetime(default=fields.Datetime.now, index=True)
-    staging_end = fields.Datetime()
+    staging_end = fields.Datetime(store=True, compute='_compute_state')
     staging_duration = fields.Float(compute='_compute_duration')
     timeout_limit = fields.Datetime(store=True, compute='_compute_timeout_limit')
     reason = fields.Text("Reason for final state (if any)")
@@ -1807,23 +1811,13 @@ class Stagings(models.Model):
     commits = fields.One2many('runbot_merge.stagings.commits', 'staging_id')
 
     statuses = fields.Binary(compute='_compute_statuses')
-    statuses_cache = fields.Text()
+    statuses_cache = fields.Text(default='{}', required=True)
 
     def write(self, vals):
         # don't allow updating the statuses_cache
         vals.pop('statuses_cache', None)
 
-        if 'state' not in vals:
-            return super().write(vals)
-
-        previously_pending = self.filtered(lambda s: s.state == 'pending')
         super().write(vals)
-        for staging in previously_pending:
-            if staging.state != 'pending':
-                super(Stagings, staging).write({
-                    'staging_end': fields.Datetime.now(),
-                    'statuses_cache': json.dumps(staging.statuses),
-                })
 
         return True
 
@@ -1851,7 +1845,7 @@ class Stagings(models.Model):
     def _search_batch_ids(self, operator, value):
         return [('staging_batch_ids.runbot_merge_batch_id', operator, value)]
 
-    @api.depends('heads')
+    @api.depends('heads', 'statuses_cache')
     def _compute_statuses(self):
         """ Fetches statuses associated with the various heads, returned as
         (repo, context, state, url)
@@ -1860,9 +1854,7 @@ class Stagings(models.Model):
         all_heads = self.mapped('head_ids')
 
         for st in self:
-            if st.statuses_cache:
-                st.statuses = json.loads(st.statuses_cache)
-                continue
+            statuses = json.loads(st.statuses_cache)
 
             commits = st.head_ids.with_prefetch(all_heads._prefetch_ids)
             st.statuses = [
@@ -1873,7 +1865,7 @@ class Stagings(models.Model):
                     status.get('target_url') or ''
                 )
                 for commit in commits
-                for context, status in json.loads(commit.statuses).items()
+                for context, status in statuses.get(commit.sha, {}).items()
             ]
 
     # only depend on staged_at as it should not get modified, but we might
@@ -1892,7 +1884,26 @@ class Stagings(models.Model):
         for staging in self:
             staging.pr_ids = staging.batch_ids.prs
 
-    def _validate(self):
+    def _notify(self, c: Commit) -> None:
+        self.env.cr.execute("""
+        UPDATE runbot_merge_stagings
+        SET statuses_cache = CASE
+            WHEN statuses_cache::jsonb->%(sha)s IS NULL
+                THEN jsonb_insert(statuses_cache::jsonb, ARRAY[%(sha)s],  %(statuses)s::jsonb)
+            ELSE statuses_cache::jsonb || jsonb_build_object(%(sha)s, %(statuses)s::jsonb)
+        END::text
+        WHERE id = any(%(ids)s)
+        """, {'sha': c.sha, 'statuses': c.statuses, 'ids': self.ids})
+        self.modified(['statuses_cache'])
+
+    @api.depends(
+        "statuses_cache",
+        "target",
+        "heads.commit_id.sha",
+        "heads.repository_id.status_ids.branch_filter",
+        "heads.repository_id.status_ids.context",
+    )
+    def _compute_state(self):
         for s in self:
             if s.state != 'pending':
                 continue
@@ -1902,8 +1913,7 @@ class Stagings(models.Model):
                 (h.commit_id.sha, h.repository_id.status_ids._for_staging(s).mapped('context'))
                 for h in s.heads
             ]
-            # maps commits to their statuses
-            cmap = {c.sha: json.loads(c.statuses) for c in s.head_ids}
+            cmap = json.loads(s.statuses_cache)
 
             update_timeout_limit = False
             st = 'success'
@@ -1920,11 +1930,12 @@ class Stagings(models.Model):
                     else:
                         assert v == 'success'
 
-            vals = {'state': st}
+            s.state = st
+            if s.state != 'pending':
+                s.staging_end = fields.Datetime.now()
             if update_timeout_limit:
-                vals['timeout_limit'] = fields.Datetime.to_string(datetime.datetime.now() + datetime.timedelta(minutes=s.target.project_id.ci_timeout))
-                _logger.debug("%s got pending status, bumping timeout to %s (%s)", self, vals['timeout_limit'], cmap)
-            s.write(vals)
+                s.timeout_limit = fields.Datetime.to_string(datetime.datetime.now() + datetime.timedelta(minutes=s.target.project_id.ci_timeout))
+                _logger.debug("%s got pending status, bumping timeout to %s (%s)", self, s.timeout_limit, cmap)
 
     def action_cancel(self):
         w = self.env['runbot_merge.stagings.cancel'].create({

From 9c51f87aed36fc1f310e3854124d8767f65d871f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 3 Jun 2024 12:59:24 +0200
Subject: [PATCH 109/192] [ADD] runbot_merge: support for non-webhook staging
 validation

Add support for the ability to validate *stagings* over RPC rather
than via webhook. This may later be expanded to PRs as well.

The core motivation for this is to avoid bouncing through github which
sometimes drops the ball on statuses, and it's frustrating to have a
staging time out because GH fucked up.

Implemented via RPC, requiring both the staging itself (by id) and the
head commit being affected, as that is necessary to know what CIs are
required for that head and correctly report cross branch on the
various PRs.

Fix #881 (kinda)
---
 runbot_merge/models/project.py              |   1 +
 runbot_merge/models/pull_requests.py        |  35 +++++--
 runbot_merge/security/security.xml          |   3 +
 runbot_merge/tests/test_basic.py            | 109 ++++++++++++--------
 runbot_merge/views/runbot_merge_project.xml |  10 ++
 5 files changed, 107 insertions(+), 51 deletions(-)

diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index dee87735..db41758b 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -33,6 +33,7 @@ class Project(models.Model):
         ('ready', "Ready PRs over split"),
     ], default="default", required=True)
     staging_statuses = fields.Boolean(default=True)
+    staging_rpc = fields.Boolean(default=False)
 
     ci_timeout = fields.Integer(
         default=60, required=True, group_operator=None,
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index a8716ba7..2055a2cf 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -18,6 +18,7 @@ import sentry_sdk
 import werkzeug
 
 from odoo import api, fields, models, tools, Command
+from odoo.exceptions import AccessError
 from odoo.osv import expression
 from odoo.tools import html_escape, Reverse
 from . import commands
@@ -1813,14 +1814,6 @@ class Stagings(models.Model):
     statuses = fields.Binary(compute='_compute_statuses')
     statuses_cache = fields.Text(default='{}', required=True)
 
-    def write(self, vals):
-        # don't allow updating the statuses_cache
-        vals.pop('statuses_cache', None)
-
-        super().write(vals)
-
-        return True
-
     @api.depends('staged_at', 'staging_end')
     def _compute_duration(self):
         for s in self:
@@ -1896,6 +1889,27 @@ class Stagings(models.Model):
         """, {'sha': c.sha, 'statuses': c.statuses, 'ids': self.ids})
         self.modified(['statuses_cache'])
 
+    def post_status(self, sha, context, status, *, target_url=None, description=None):
+        if not self.env.user.has_group('runbot_merge.status'):
+            raise AccessError("You are not allowed to post a status.")
+
+        for s in self:
+            if not s.target.project_id.staging_rpc:
+                continue
+
+            if not any(c.commit_id.sha == sha for c in s.commits):
+                raise ValueError(f"Staging {s.id} does not have the commit {sha}")
+
+            st = json.loads(s.statuses_cache)
+            st.setdefault(sha, {})[context] = {
+                'state': status,
+                'target_url': target_url,
+                'description': description,
+            }
+            s.statuses_cache = json.dumps(st)
+
+        return True
+
     @api.depends(
         "statuses_cache",
         "target",
@@ -2006,17 +2020,18 @@ class Stagings(models.Model):
             })
             return True
 
-        # single batch => the staging is an unredeemable failure
+        # single batch => the staging is an irredeemable failure
         if self.state != 'failure':
             # timed out, just mark all PRs (wheee)
             self.fail('timed out (>{} minutes)'.format(self.target.project_id.ci_timeout))
             return False
 
+        staging_statuses = json.loads(self.statuses_cache)
         # try inferring which PR failed and only mark that one
         for head in self.heads:
             required_statuses = set(head.repository_id.status_ids._for_staging(self).mapped('context'))
 
-            statuses = json.loads(head.commit_id.statuses or '{}')
+            statuses = staging_statuses.get(head.commit_id.sha, {})
             reason = next((
                 ctx for ctx, result in statuses.items()
                 if ctx in required_statuses
diff --git a/runbot_merge/security/security.xml b/runbot_merge/security/security.xml
index 62e1f323..d1ea1c79 100644
--- a/runbot_merge/security/security.xml
+++ b/runbot_merge/security/security.xml
@@ -5,4 +5,7 @@
     <record model="res.groups" id="base.group_system">
         <field name="implied_ids" eval="[(4, ref('runbot_merge.group_admin'))]"/>
     </record>
+    <record model="res.groups" id="status">
+        <field name="name">Mergebot Status Sender</field>
+    </record>
 </odoo>
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index afa7a3a9..9299738a 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -24,6 +24,41 @@ def repo(env, project, make_repo, users, setreviewers):
     setreviewers(*project.repo_ids)
     return r
 
+@pytest.fixture(autouse=True, params=["statuses", "rpc"])
+def stagings(request, env, project, repo):
+    """Hook in support for validating stagings via RPC calls instead of CI
+    webhooks. Transparent for the tests as long as they send statuses to
+    symbolic refs (branch names) rather than commits, although commits *would*
+    probably be doable (look up the head for the commit, then what staging it's
+    part of)
+    """
+    if request.param == "statuses":
+            yield
+    else:
+        env['res.users'].browse([env._uid]).write({
+            "groups_id": [(4, env.ref("runbot_merge.status").id, {})]
+        })
+        project.write({
+            "staging_rpc": True,
+            "staging_statuses": False,
+        })
+        RepoType = type(repo)
+        # apparently side_effect + wraps on unbound method don't work correctly,
+        # the wrapped method does get called when returning DEFAULT but *the
+        # instance (subject) is not sent along for the ride* so the call fails.
+        post_status = RepoType.post_status
+        def _post_status(repo, ref, status, context='default', **kw):
+            if not ref.startswith(('staging.', 'heads/staging.')):
+                return post_status(repo, ref, status, context, **kw)
+
+            c = repo.commit(ref)
+            branchname = ref.removeprefix('staging.').removeprefix('heads/staging.')
+            env['runbot_merge.stagings'].search([('target.name', '=', branchname)])\
+                .post_status(c.id, context, status, **kw)
+
+        with mock.patch.object(RepoType, "post_status", _post_status):
+            yield
+
 def test_trivial_flow(env, repo, page, users, config):
     # create base branch
     with repo:
@@ -81,12 +116,10 @@ def test_trivial_flow(env, repo, page, users, config):
     assert pr_page(page, pr).cssselect('.alert-primary')
 
     with repo:
-        # get head of staging branch
-        staging_head = repo.commit('heads/staging.master')
-        repo.post_status(staging_head.id, 'success', 'ci/runbot', target_url='http://foo.com/pog')
-        repo.post_status(staging_head.id, 'success', 'legal/cla')
+        repo.post_status('staging.master', 'success', 'ci/runbot', target_url='http://foo.com/pog')
+        repo.post_status('staging.master', 'success', 'legal/cla')
         # the should not block the merge because it's not part of the requirements
-        repo.post_status(staging_head.id, 'failure', 'ci/lint', target_url='http://ignored.com/whocares')
+        repo.post_status('staging.master', 'failure', 'ci/lint', target_url='http://ignored.com/whocares')
     # need to store this because after the crons have run the staging will
     # have succeeded and been disabled
     st = pr_id.staging_id
@@ -404,18 +437,16 @@ def test_staging_ongoing(env, repo, config):
     ])
     assert p_2.state == 'ready', "PR2 should not have been staged since there is a pending staging for master"
 
-    staging_head = repo.commit('heads/staging.master')
     with repo:
-        repo.post_status(staging_head.id, 'success', 'ci/runbot')
-        repo.post_status(staging_head.id, 'success', 'legal/cla')
+        repo.post_status('staging.master', 'success', 'ci/runbot')
+        repo.post_status('staging.master', 'success', 'legal/cla')
     env.run_crons()
     assert pr1.state == 'merged'
     assert p_2.staging_id
 
-    staging_head = repo.commit('heads/staging.master')
     with repo:
-        repo.post_status(staging_head.id, 'success', 'ci/runbot')
-        repo.post_status(staging_head.id, 'success', 'legal/cla')
+        repo.post_status('staging.master', 'success', 'ci/runbot')
+        repo.post_status('staging.master', 'success', 'legal/cla')
     env.run_crons()
     assert p_2.state == 'merged'
 
@@ -570,7 +601,7 @@ def test_timeout_bump_on_pending(env, repo, config):
     old_timeout = odoo.fields.Datetime.to_string(datetime.datetime.now() - datetime.timedelta(days=15))
     st.timeout_limit = old_timeout
     with repo:
-        repo.post_status(repo.commit('heads/staging.master').id, 'pending', 'ci/runbot')
+        repo.post_status('staging.master', 'pending', 'ci/runbot')
     env.run_crons('runbot_merge.process_updated_commits')
     assert st.timeout_limit > old_timeout
 
@@ -591,11 +622,10 @@ def test_staging_ci_failure_single(env, repo, users, config, page):
     pr_id = to_pr(env, pr)
     assert pr_id.staging_id
 
-    staging_head = repo.commit('heads/staging.master')
     with repo:
-        repo.post_status(staging_head.id, 'failure', 'a/b')
-        repo.post_status(staging_head.id, 'success', 'legal/cla')
-        repo.post_status(staging_head.id, 'failure', 'ci/runbot') # stable genius
+        repo.post_status('staging.master', 'failure', 'a/b')
+        repo.post_status('staging.master', 'success', 'legal/cla')
+        repo.post_status('staging.master', 'failure', 'ci/runbot') # stable genius
     env.run_crons()
     assert pr_id.state == 'error'
 
@@ -636,8 +666,8 @@ def test_ff_failure(env, repo, config, page):
     # report staging success & run cron to merge
     staging = repo.commit('heads/staging.master')
     with repo:
-        repo.post_status(staging.id, 'success', 'legal/cla')
-        repo.post_status(staging.id, 'success', 'ci/runbot')
+        repo.post_status('staging.master', 'success', 'legal/cla')
+        repo.post_status('staging.master', 'success', 'ci/runbot')
     env.run_crons()
 
     assert st.reason == 'update is not a fast forward'
@@ -933,8 +963,8 @@ def test_forward_port(env, repo, config):
     st = repo.commit('staging.master')
 
     with repo:
-        repo.post_status(st.id, 'success', 'legal/cla')
-        repo.post_status(st.id, 'success', 'ci/runbot')
+        repo.post_status('staging.master', 'success', 'legal/cla')
+        repo.post_status('staging.master', 'success', 'ci/runbot')
     env.run_crons()
 
     h = repo.commit('master')
@@ -1158,8 +1188,8 @@ class TestRetry:
         ]).staging_id
 
         staging_head = repo.commit('heads/staging.master')
-        repo.post_status(staging_head.id, 'success', 'legal/cla')
-        repo.post_status(staging_head.id, 'failure', 'ci/runbot')
+        repo.post_status('staging.master', 'success', 'legal/cla')
+        repo.post_status('staging.master', 'failure', 'ci/runbot')
         env.run_crons()
         pr = env['runbot_merge.pull_requests'].search([
             ('repository.name', '=', repo.name),
@@ -1178,8 +1208,8 @@ class TestRetry:
 
         staging_head2 = repo.commit('heads/staging.master')
         assert staging_head2 != staging_head
-        repo.post_status(staging_head2.id, 'success', 'legal/cla')
-        repo.post_status(staging_head2.id, 'success', 'ci/runbot')
+        repo.post_status('staging.master', 'success', 'legal/cla')
+        repo.post_status('staging.master', 'success', 'ci/runbot')
         env.run_crons()
         assert pr.state == 'merged'
 
@@ -1202,8 +1232,8 @@ class TestRetry:
 
         staging_head = repo.commit('heads/staging.master')
         with repo:
-            repo.post_status(staging_head.id, 'success', 'legal/cla')
-            repo.post_status(staging_head.id, 'failure', 'ci/runbot')
+            repo.post_status('staging.master', 'success', 'legal/cla')
+            repo.post_status('staging.master', 'failure', 'ci/runbot')
         env.run_crons()
         assert env['runbot_merge.pull_requests'].search([
             ('repository.name', '=', repo.name),
@@ -1221,8 +1251,8 @@ class TestRetry:
         staging_head2 = repo.commit('heads/staging.master')
         assert staging_head2 != staging_head
         with repo:
-            repo.post_status(staging_head2.id, 'success', 'legal/cla')
-            repo.post_status(staging_head2.id, 'success', 'ci/runbot')
+            repo.post_status('staging.master', 'success', 'legal/cla')
+            repo.post_status('staging.master', 'success', 'ci/runbot')
         env.run_crons()
         assert env['runbot_merge.pull_requests'].search([
             ('repository.name', '=', repo.name),
@@ -1297,8 +1327,8 @@ class TestRetry:
 
         staging_head = repo.commit('heads/staging.master')
         with repo:
-            repo.post_status(staging_head.id, 'success', 'legal/cla')
-            repo.post_status(staging_head.id, 'failure', 'ci/runbot')
+            repo.post_status('staging.master', 'success', 'legal/cla')
+            repo.post_status('staging.master', 'failure', 'ci/runbot')
         env.run_crons()
         pr = env['runbot_merge.pull_requests'].search([
             ('repository.name', '=', repo.name),
@@ -1357,8 +1387,8 @@ class TestMergeMethod:
             "dummy commit aside, the previous master's tip should be the sole parent of the staging commit"
 
         with repo:
-            repo.post_status(staging.id, 'success', 'legal/cla')
-            repo.post_status(staging.id, 'success', 'ci/runbot')
+            repo.post_status('staging.master', 'success', 'legal/cla')
+            repo.post_status('staging.master', 'success', 'ci/runbot')
         env.run_crons()
         pr = env['runbot_merge.pull_requests'].search([
             ('repository.name', '=', repo.name),
@@ -2367,10 +2397,9 @@ class TestPRUpdate(object):
         assert pr.state == 'ready'
         assert pr.staging_id
 
-        h = repo.commit('heads/staging.master').id
         with repo:
-            repo.post_status(h, 'success', 'legal/cla')
-            repo.post_status(h, 'failure', 'ci/runbot')
+            repo.post_status('staging.master', 'success', 'legal/cla')
+            repo.post_status('staging.master', 'failure', 'ci/runbot')
         env.run_crons()
         assert not pr.staging_id
         assert pr.state == 'error'
@@ -3052,19 +3081,17 @@ class TestBatching(object):
         assert len(sp) == 1
 
         # This is the failing PR!
-        h = repo.commit('heads/staging.master').id
         with repo:
-            repo.post_status(h, 'failure', 'ci/runbot')
-            repo.post_status(h, 'success', 'legal/cla')
+            repo.post_status('staging.master', 'failure', 'ci/runbot')
+            repo.post_status('staging.master', 'success', 'legal/cla')
         env.run_crons()
         assert pr1.state == 'error'
 
         assert pr2.staging_id
 
-        h = repo.commit('heads/staging.master').id
         with repo:
-            repo.post_status(h, 'success', 'ci/runbot')
-            repo.post_status(h, 'success', 'legal/cla')
+            repo.post_status('staging.master', 'success', 'ci/runbot')
+            repo.post_status('staging.master', 'success', 'legal/cla')
         env.run_crons('runbot_merge.process_updated_commits', 'runbot_merge.merge_cron', 'runbot_merge.staging_cron')
         assert pr2.state == 'merged'
 
diff --git a/runbot_merge/views/runbot_merge_project.xml b/runbot_merge/views/runbot_merge_project.xml
index d7b9324f..68eb28cc 100644
--- a/runbot_merge/views/runbot_merge_project.xml
+++ b/runbot_merge/views/runbot_merge_project.xml
@@ -30,6 +30,16 @@
                             <field name="github_email" readonly="0"
                                    help="Identity when creating new commits, defaults to public email, falls back to primary email."/>
                             <field name="secret"/>
+                            <span attrs="{'invisible': [
+                                '|',
+                                    ('staging_statuses', '=', False),
+                                    ('staging_rpc', '=', False),
+                            ]}" class="alert alert-warning" role="alert">
+                                Avoid overlaps between GH and RPC as the older
+                                GH statuses may overwrite more recent RPC statuses.
+                            </span>
+                            <field name="staging_statuses" string="Validate via GH statuses"/>
+                            <field name="staging_rpc" string="Validate via direct RPC"/>
                         </group>
                         <group>
                             <field name="staging_enabled" widget="boolean_toggle"/>

From 98aaa9107ffa3de1844ff13238c530d8448ce484 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 3 Jun 2024 15:29:47 +0200
Subject: [PATCH 110/192] [CHG] forwardport: notify the outstanding
 forwardports rather than source

I have been convinced that this might be an improvement to the affairs
of the people: originally the message was sent to the source PR so we
wouldn't have to ping the author & reviewer and to limit the amount of
spam, *however*:

- we ended up adding pings anyway
- it also pings the followers of the source PR
- it increases the size of the original discussion (especially if was
- originally long)
- it adds steps to fixing the issue as you need to bounce from the
  source to the forward ports

Note that this might still notify a lot of people as they might be
made followers of the forward ports automatically, and it increases
the messaging load of the forwardbot significantly. But we'll see how
things go. Worst case scenario, we can revert it back.

Fixes #836
---
 forwardport/models/project.py                 | 19 ++---
 forwardport/tests/test_simple.py              | 69 +++++++++++++++----
 ..._merge.pull_requests.feedback.template.csv |  6 +-
 3 files changed, 67 insertions(+), 27 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index a4099d05..1839979d 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -553,18 +553,13 @@ stderr:
             if source.merge_date > (cutoff_dt - backoff):
                 continue
             source.reminder_backoff_factor += 1
-            self.env.ref('runbot_merge.forwardport.reminder')._send(
-                repository=source.repository,
-                pull_request=source.number,
-                token_field='fp_github_token',
-                format_args={
-                    'pr': source,
-                    'outstanding': ''.join(
-                        f'\n- {pr.display_name}'
-                        for pr in sorted(prs, key=lambda p: p.number)
-                    ),
-                }
-            )
+            for pr in prs:
+                self.env.ref('runbot_merge.forwardport.reminder')._send(
+                    repository=pr.repository,
+                    pull_request=pr.number,
+                    token_field='fp_github_token',
+                    format_args={'pr': pr, 'source': source},
+                )
 
 class Stagings(models.Model):
     _inherit = 'runbot_merge.stagings'
diff --git a/forwardport/tests/test_simple.py b/forwardport/tests/test_simple.py
index c086bd9b..e7214871 100644
--- a/forwardport/tests/test_simple.py
+++ b/forwardport/tests/test_simple.py
@@ -6,7 +6,7 @@ from datetime import datetime, timedelta
 
 import pytest
 
-from utils import seen, Commit, make_basic, REF_PATTERN, MESSAGE_TEMPLATE, validate_all, part_of, to_pr
+from utils import seen, Commit, make_basic, REF_PATTERN, MESSAGE_TEMPLATE, validate_all, part_of, to_pr, re_matches
 
 FMT = '%Y-%m-%d %H:%M:%S'
 FAKE_PREV_WEEK = (datetime.now() + timedelta(days=1)).strftime(FMT)
@@ -132,10 +132,20 @@ def test_straightforward_flow(env, config, make_repo, users):
         (users['reviewer'], 'hansen r+ rebase-ff'),
         seen(env, pr, users),
         (users['user'], 'Merge method set to rebase and fast-forward.'),
-        (users['user'], '@%s @%s this pull request has forward-port PRs awaiting action (not merged or closed):\n- %s' % (
-            users['other'], users['reviewer'],
-            '\n- '.join((pr1 | pr2).mapped('display_name'))
-        )),
+    ]
+    pr1_remote = prod.get_pr(pr1.number)
+    assert pr1_remote.comments == [
+        seen(env, pr1_remote, users),
+        (users['user'], """\
+This PR targets b and is part of the forward-port chain. Further PRs will be created up to c.
+
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+"""),
+        (users['user'], "@%s @%s this forward port of %s is awaiting action (not merged or closed)." % (
+            users['other'],
+            users['reviewer'],
+            pr0.display_name,
+        ))
     ]
 
     assert pr0_ == pr0
@@ -168,6 +178,11 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
             users['other'], users['reviewer'],
             pr1.display_name,
     )),
+        (users['user'], "@%s @%s this forward port of %s is awaiting action (not merged or closed)." % (
+            users['other'],
+            users['reviewer'],
+            pr0.display_name,
+        ))
     ]
     with prod:
         prod.post_status(pr2.head, 'success', 'ci/runbot')
@@ -231,7 +246,7 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
     assert other_user_repo.get_ref(pr.ref) == p_1
 
     # should have deleted all PR branches
-    pr1_ref = prod.get_pr(pr1.number).ref
+    pr1_ref = pr1_remote.ref
     with pytest.raises(AssertionError, match='Not Found'):
         other.get_ref(pr1_ref)
 
@@ -326,27 +341,57 @@ def test_empty(env, config, make_repo, users):
 
     awaiting = (
         users['other'],
-        '@%s @%s this pull request has forward-port PRs awaiting action (not merged or closed):\n- %s' % (
+        '@%s @%s this forward port of %s is awaiting action (not merged or closed).' % (
             users['user'], users['reviewer'],
-            fail_id.display_name
+            pr1_id.display_name
         )
     )
+    conflict = (users['user'], re_matches(
+        fr"""@{users['user']} @{users['reviewer']} cherrypicking of pull request {pr1_id.display_name} failed\.
+
+stdout:
+```
+.*
+```
+
+stderr:
+```
+.*
+```
+
+Either perform the forward-port manually \(and push to this branch, proceeding as usual\) or close this PR \(maybe\?\)\.
+
+In the former case, you may want to edit this PR message as well\.
+
+:warning: after resolving this conflict, you will need to merge it via @{project.github_prefix}\.
+
+More info at https://github\.com/odoo/odoo/wiki/Mergebot#forward-port
+""", re.DOTALL))
     assert pr1.comments == [
         (users['reviewer'], 'hansen r+'),
         seen(env, pr1, users),
+    ]
+    fail_pr = prod.get_pr(fail_id.number)
+    assert fail_pr.comments == [
+        seen(env, fail_pr, users),
+        conflict,
         awaiting,
         awaiting,
-    ], "each cron run should trigger a new message on the ancestor"
+    ], "each cron run should trigger a new message"
     # check that this stops if we close the PR
     with prod:
-        prod.get_pr(fail_id.number).close()
+        fail_pr.close()
     env.run_crons('forwardport.reminder', 'runbot_merge.feedback_cron', context={'forwardport_updated_before': FAKE_PREV_WEEK})
     assert pr1.comments == [
         (users['reviewer'], 'hansen r+'),
         seen(env, pr1, users),
-        awaiting,
-        awaiting,
     ]
+    assert fail_pr.comments == [
+        seen(env, fail_pr, users),
+        conflict,
+        awaiting,
+        awaiting,
+    ], "each cron run should trigger a new message"
 
 def test_partially_empty(env, config, make_repo):
     """ Check what happens when only some commits of the PR are now empty
diff --git a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
index 5fe768ae..eab2a241 100644
--- a/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
+++ b/runbot_merge/data/runbot_merge.pull_requests.feedback.template.csv
@@ -131,10 +131,10 @@ runbot_merge.forwardport.failure.conflict,"{pr.ping}the next pull request ({new.
 pr: the pr which was just forward ported
 new: the new forward-port
 footer: some footer text"
-runbot_merge.forwardport.reminder,{pr.ping}this pull request has forward-port PRs awaiting action (not merged or closed):{outstanding},"Comment when a forward port has outstanding (not merged or closed) descendants
+runbot_merge.forwardport.reminder,{pr.ping}this forward port of {source.display_name} is awaiting action (not merged or closed).,"Comment when a forward port has outstanding (not merged or closed) descendants
 
-pr: the original (source) PR
-outstanding: markdown-formatted list of outstanding PRs"
+pr: the forward-port
+source: the source PR"
 runbot_merge.forwardport.failure,"{pr.ping}cherrypicking of pull request {pr.source_id.display_name} failed.
 {commits}{stdout}{stderr}
 Either perform the forward-port manually (and push to this branch, proceeding as usual) or close this PR (maybe?).

From c1e2e5a2e0dea4f9bfd53311d08f82aa90ac019c Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 4 Jun 2024 12:15:29 +0200
Subject: [PATCH 111/192] [REF] forwardport: update re_matches to not use a
 regex

Using a regex as the pattern is quite frustrating due to all the
escaping necessary, which in this refactoring I found out I'd missed,
multiple times.

Convert the pattern to something bespoke but not too complicated, we
may want to add anchoring support and a bit more finesse and the
future but for now straightforward "holes" seem to work well. I've
added support for capturing and even named groups even if this as yet
unnecessary and unused.

Fixes #861

[^1]: https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_assertrepr_compare
---
 forwardport/tests/test_batches.py   | 18 +++++++++---------
 forwardport/tests/test_conflicts.py | 28 ++++++++++++++--------------
 forwardport/tests/test_simple.py    | 20 ++++++++++----------
 forwardport/tests/test_updates.py   | 24 ++++++++++++++----------
 mergebot_test_utils/utils.py        | 26 +++++++++++++++++---------
 runbot_merge/tests/test_basic.py    | 10 +++++-----
 6 files changed, 69 insertions(+), 57 deletions(-)

diff --git a/forwardport/tests/test_batches.py b/forwardport/tests/test_batches.py
index ad1c6f8b..b0b7155a 100644
--- a/forwardport/tests/test_batches.py
+++ b/forwardport/tests/test_batches.py
@@ -1,6 +1,6 @@
 import re
 
-from utils import Commit, make_basic, to_pr, seen, re_matches
+from utils import Commit, make_basic, to_pr, seen, matches
 
 
 def test_single_updated(env, config, make_repo):
@@ -381,26 +381,26 @@ def test_add_to_forward_port_conflict(env, config, make_repo, users):
     assert pr2_c.comments == [
         seen(env, pr2_c, users),
         # should have conflicts
-        (users['user'], re_matches(r"""@{user} cherrypicking of pull request {previous.display_name} failed\.
+        (users['user'], matches("""@{user} cherrypicking of pull request {previous.display_name} failed.
 
 stdout:
 ```
 Auto-merging b
-CONFLICT \(add/add\): Merge conflict in b
+CONFLICT (add/add): Merge conflict in b
 
 ```
 
 stderr:
 ```
-.*
+$$
 ```
 
-Either perform the forward-port manually \(and push to this branch, proceeding as usual\) or close this PR \(maybe\?\)\.
+Either perform the forward-port manually (and push to this branch, proceeding as usual) or close this PR (maybe?).
 
-In the former case, you may want to edit this PR message as well\.
+In the former case, you may want to edit this PR message as well.
 
-:warning: after resolving this conflict, you will need to merge it via @{project.github_prefix}\.
+:warning: after resolving this conflict, you will need to merge it via @{project.github_prefix}.
 
-More info at https://github\.com/odoo/odoo/wiki/Mergebot#forward-port
-""".format(project=project, previous=pr2_b_id, **users), re.DOTALL))
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+""".format(project=project, previous=pr2_b_id, **users)))
     ]
diff --git a/forwardport/tests/test_conflicts.py b/forwardport/tests/test_conflicts.py
index 4cad2c8c..eed2bcc0 100644
--- a/forwardport/tests/test_conflicts.py
+++ b/forwardport/tests/test_conflicts.py
@@ -2,7 +2,7 @@ import re
 import time
 from operator import itemgetter
 
-from utils import make_basic, Commit, validate_all, re_matches, seen, REF_PATTERN, to_pr
+from utils import make_basic, Commit, validate_all, matches, seen, REF_PATTERN, to_pr
 
 
 def test_conflict(env, config, make_repo, users):
@@ -59,39 +59,39 @@ def test_conflict(env, config, make_repo, users):
     assert prod.read_tree(c) == {
         'f': 'c',
         'g': 'a',
-        'h': re_matches(r'''<<<\x3c<<< HEAD
+        'h': matches('''<<<\x3c<<< HEAD
 a
-|||||||| parent of [\da-f]{7,}.*
+||||||| parent of $$ (temp)
 =======
 xxx
->>>\x3e>>> [\da-f]{7,}.*
+>>>\x3e>>> $$ (temp)
 '''),
     }
     assert prc.comments == [
         seen(env, prc, users),
-        (users['user'], re_matches(
-fr'''@{users['user']} @{users['reviewer']} cherrypicking of pull request {pra_id.display_name} failed\.
+        (users['user'], matches(
+f'''@{users['user']} @{users['reviewer']} cherrypicking of pull request {pra_id.display_name} failed.
 
 stdout:
 ```
 Auto-merging h
-CONFLICT \(add/add\): Merge conflict in h
+CONFLICT (add/add): Merge conflict in h
 
 ```
 
 stderr:
 ```
-.*
+$$
 ```
 
-Either perform the forward-port manually \(and push to this branch, proceeding as usual\) or close this PR \(maybe\?\)\.
+Either perform the forward-port manually (and push to this branch, proceeding as usual) or close this PR (maybe?).
 
-In the former case, you may want to edit this PR message as well\.
+In the former case, you may want to edit this PR message as well.
 
-:warning: after resolving this conflict, you will need to merge it via @{project.github_prefix}\.
+:warning: after resolving this conflict, you will need to merge it via @{project.github_prefix}.
 
-More info at https://github\.com/odoo/odoo/wiki/Mergebot#forward-port
-''', re.DOTALL))
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+'''))
     ]
 
     prb = prod.get_pr(prb_id.number)
@@ -372,7 +372,7 @@ b
 
     assert pr2.comments == [
         seen(env, pr2, users),
-        (users['user'], re_matches(r'@%s @%s .*CONFLICT' % (users['user'], users['reviewer']), re.DOTALL)),
+        (users['user'], matches('@%s @%s $$CONFLICT' % (users['user'], users['reviewer']))),
         (users['reviewer'], 'hansen r+'),
         (users['user'], f"@{users['user']} @{users['reviewer']} unable to stage: "
                         "All commits must have author and committer email, "
diff --git a/forwardport/tests/test_simple.py b/forwardport/tests/test_simple.py
index e7214871..901b93a7 100644
--- a/forwardport/tests/test_simple.py
+++ b/forwardport/tests/test_simple.py
@@ -6,7 +6,7 @@ from datetime import datetime, timedelta
 
 import pytest
 
-from utils import seen, Commit, make_basic, REF_PATTERN, MESSAGE_TEMPLATE, validate_all, part_of, to_pr, re_matches
+from utils import seen, Commit, make_basic, REF_PATTERN, MESSAGE_TEMPLATE, validate_all, part_of, to_pr, matches
 
 FMT = '%Y-%m-%d %H:%M:%S'
 FAKE_PREV_WEEK = (datetime.now() + timedelta(days=1)).strftime(FMT)
@@ -346,27 +346,27 @@ def test_empty(env, config, make_repo, users):
             pr1_id.display_name
         )
     )
-    conflict = (users['user'], re_matches(
-        fr"""@{users['user']} @{users['reviewer']} cherrypicking of pull request {pr1_id.display_name} failed\.
+    conflict = (users['user'], matches(
+        f"""@{users['user']} @{users['reviewer']} cherrypicking of pull request {pr1_id.display_name} failed.
 
 stdout:
 ```
-.*
+$$
 ```
 
 stderr:
 ```
-.*
+$$
 ```
 
-Either perform the forward-port manually \(and push to this branch, proceeding as usual\) or close this PR \(maybe\?\)\.
+Either perform the forward-port manually (and push to this branch, proceeding as usual) or close this PR (maybe?).
 
-In the former case, you may want to edit this PR message as well\.
+In the former case, you may want to edit this PR message as well.
 
-:warning: after resolving this conflict, you will need to merge it via @{project.github_prefix}\.
+:warning: after resolving this conflict, you will need to merge it via @{project.github_prefix}.
 
-More info at https://github\.com/odoo/odoo/wiki/Mergebot#forward-port
-""", re.DOTALL))
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+"""))
     assert pr1.comments == [
         (users['reviewer'], 'hansen r+'),
         seen(env, pr1, users),
diff --git a/forwardport/tests/test_updates.py b/forwardport/tests/test_updates.py
index 3a6b4a11..0dedf9dc 100644
--- a/forwardport/tests/test_updates.py
+++ b/forwardport/tests/test_updates.py
@@ -6,7 +6,7 @@ import re
 
 import pytest
 
-from utils import seen, re_matches, Commit, make_basic, to_pr
+from utils import seen, matches, Commit, make_basic, to_pr
 
 
 @pytest.mark.parametrize("merge_parent", [False, True])
@@ -433,12 +433,12 @@ def test_subsequent_conflict(env, make_repo, config, users):
     assert repo.read_tree(repo.commit(pr3_id.head)) == {
         'f': 'c',
         'g': 'a',
-        'h': re_matches(r'''<<<\x3c<<< HEAD
+        'h': matches('''<<<\x3c<<< HEAD
 a
-|||||||| parent of [\da-f]{7,}.*
+||||||| parent of $$ (temp)
 =======
 conflict!
->>>\x3e>>> [\da-f]{7,}.*
+>>>\x3e>>> $$ (temp)
 '''),
         'x': '0',
     }
@@ -458,18 +458,22 @@ conflict!
     # 1. link to status page
     # 2. forward-port chain thing
     assert repo.get_pr(pr3_id.number).comments[2:] == [
-        (users['user'], re_matches(f'''\
+        (users['user'], matches(f'''\
 @{users['user']} @{users['reviewer']} WARNING: the update of {pr2_id.display_name} to {pr2_id.head} has caused a \
 conflict in this pull request, data may have been lost.
 
 stdout:
-```.*?
-CONFLICT \(add/add\): Merge conflict in h.*?
+```
+Auto-merging h
+CONFLICT (add/add): Merge conflict in h
 ```
 
 stderr:
 ```
-\\d{{2}}:\\d{{2}}:\\d{{2}}.\\d+ .* {pr2_id.head}
-error: could not apply [0-9a-f]+\\.\\.\\. newfiles
-''', re.DOTALL))
+$$:$$:$$.$$ {pr2_id.head}
+error: could not apply $$... newfiles
+hint: $$
+----------
+status:
+'''))
     ]
diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index cba96114..0d0f7dff 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -42,18 +42,26 @@ def _simple_init(repo):
     prx = repo.make_pr(title='title', body='body', target='master', head=c2)
     return prx
 
-class re_matches:
+class matches(str):
+    # necessary so str.__new__ does not freak out on `flags`
+    def __new__(cls, pattern, flags=0):
+        return super().__new__(cls, pattern)
+
     def __init__(self, pattern, flags=0):
-        self._r = re.compile(pattern, flags)
+        p, n = re.subn(
+            # `re.escape` will escape the `$`, so we need to handle that...
+            # maybe it should not be $?
+            r'\\\$(\w*?)\\\$',
+            lambda m: f'(?P<{m[1]}>.*?)' if m[1] else '(.*?)',
+            re.escape(self),
+        )
+        assert n, f"matches' pattern should have at least one placeholder, found none in\n{pattern}"
+        self._r = re.compile(p, flags | re.DOTALL)
 
     def __eq__(self, text):
-        return self._r.match(text)
-
-    def __str__(self):
-        return re.sub(r'\\(.)', r'\1', self._r.pattern)
-
-    def __repr__(self):
-        return repr(str(self))
+        if not isinstance(text, str):
+            return NotImplemented
+        return self._r.search(text)
 
 def seen(env, pr, users):
     url = to_pr(env, pr).url
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 9299738a..d43368e9 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -10,7 +10,7 @@ import requests
 from lxml import html
 
 import odoo
-from utils import _simple_init, seen, re_matches, get_partner, Commit, pr_page, to_pr, part_of, ensure_one
+from utils import _simple_init, seen, matches, get_partner, Commit, pr_page, to_pr, part_of, ensure_one
 
 
 @pytest.fixture
@@ -174,7 +174,7 @@ def test_trivial_flow(env, repo, page, users, config):
             ('', 'Reviewer'),
         ]),
         # staging succeeded
-        (re_matches(r'.*'), f'<p>staging {st.id} succeeded</p>', [
+        (matches('$$'), f'<p>staging {st.id} succeeded</p>', [
             # set merge date
             (False, pr_id.merge_date + 'Z'),
             # updated state
@@ -676,7 +676,7 @@ def test_ff_failure(env, repo, config, page):
     _new, prev = doc.cssselect('li.staging')
 
     assert 'bg-gray-lighter' in prev.classes, "ff failure is ~ cancelling"
-    assert prev.get('title') == re_matches('fast forward failed \(update is not a fast forward\)')
+    assert 'fast forward failed (update is not a fast forward)' in prev.get('title')
 
     assert env['runbot_merge.pull_requests'].search([
         ('repository.name', '=', repo.name),
@@ -1029,7 +1029,7 @@ def test_rebase_failure(env, repo, users, config):
     assert pr_a.comments == [
         (users['reviewer'], 'hansen r+'),
         seen(env, pr_a, users),
-        (users['user'], re_matches(r'^Unable to stage PR')),
+        (users['user'], matches('Unable to stage PR')),
     ]
     assert pr_b.comments == [
         (users['reviewer'], 'hansen r+'),
@@ -3968,7 +3968,7 @@ class TestInfrastructure:
         assert repo.get_ref('heads/master') == m1
 
 def node(name, *children):
-    assert type(name) in (str, re_matches)
+    assert type(name) in (str, matches)
     return name, frozenset(children)
 def log_to_node(log):
     log = list(log)

From cc9f2392613f4d608db073d8b960fa7042cb01c8 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 4 Jun 2024 14:18:36 +0200
Subject: [PATCH 112/192] [IMP] forwardport: use ORT and zdiff3 for cherrypicks

- zdiff3 should provide better conflict annotations than diff3
- ORT might also provide less conflicts period
- always perform cherrypicks without rename limit, since we're
  re-trying every failure without rename limit, it seems like
  unnecessary work, assuming git only ever looks as far as it needs
- also enable copy support maybe...

Fixes #827
---
 forwardport/models/project.py     | 34 ++++++++++++++++---------------
 forwardport/tests/test_updates.py | 14 +++++++++----
 2 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 1839979d..1ce6abb7 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -423,9 +423,13 @@ class PullRequests(models.Model):
             # switch back to the PR branch
             conf.checkout(fp_branch_name)
             # cherry-pick the squashed commit to generate the conflict
-            conf.with_params('merge.renamelimit=0', 'merge.conflictstyle=diff3')\
+            conf.with_params(
+                'merge.renamelimit=0',
+                'merge.renames=copies',
+                'merge.conflictstyle=zdiff3'
+            )\
                 .with_config(check=False)\
-                .cherry_pick(squashed, no_commit=True)
+                .cherry_pick(squashed, no_commit=True, strategy="ort")
             status = conf.stdout().status(short=True, untracked_files='no').stdout.decode()
             if err.strip():
                 err = err.rstrip() + '\n----------\nstatus:\n' + status
@@ -459,7 +463,7 @@ stderr:
         logger = _logger.getChild(str(self.id)).getChild('cherrypick')
 
         # original head so we can reset
-        prev = original_head = working_copy.stdout().rev_parse('HEAD').stdout.decode().strip()
+        original_head = working_copy.stdout().rev_parse('HEAD').stdout.decode().strip()
 
         commits = self.commits()
         logger.info("%s: copy %s commits to %s\n%s", self, len(commits), original_head, '\n'.join(
@@ -467,6 +471,13 @@ stderr:
             for c in commits
         ))
 
+        conf_base = working_copy.with_params(
+            'merge.renamelimit=0',
+            'merge.renames=copies',
+        ).with_config(
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            check=False
+        )
         for commit in commits:
             commit_sha = commit['sha']
             # config (global -c) or commit options don't really give access to
@@ -481,19 +492,10 @@ stderr:
             }
             configured = working_copy.with_config(env=env)
 
-            conf = working_copy.with_config(
-                env={**env, 'GIT_TRACE': 'true'},
-                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                check=False
-            )
+            conf = conf_base.with_config(env={**env, 'GIT_TRACE': 'true'})
             # first try with default / low renamelimit
-            r = conf.cherry_pick(commit_sha)
+            r = conf.cherry_pick(commit_sha, strategy="ort")
             logger.debug("Cherry-picked %s: %s\n%s\n%s", commit_sha, r.returncode, r.stdout.decode(), _clean_rename(r.stderr.decode()))
-            if r.returncode:
-                # if it failed, retry with high renamelimit
-                configured.reset('--hard', prev)
-                r = conf.with_params('merge.renamelimit=0').cherry_pick(commit_sha)
-                logger.debug("Cherry-picked %s (renamelimit=0): %s\n%s\n%s", commit_sha, r.returncode, r.stdout.decode(), _clean_rename(r.stderr.decode()))
 
             if r.returncode: # pick failed, reset and bail
                 # try to log inflateInit: out of memory errors as warning, they
@@ -516,8 +518,8 @@ stderr:
             configured \
                 .with_config(input=str(msg).encode())\
                 .commit(amend=True, file='-')
-            prev = configured.stdout().rev_parse('HEAD').stdout.decode()
-            logger.info('%s: success -> %s', commit_sha, prev)
+            result = configured.stdout().rev_parse('HEAD').stdout.decode()
+            logger.info('%s: success -> %s', commit_sha, result)
 
     def _make_fp_message(self, commit):
         cmap = json.loads(self.commits_map)
diff --git a/forwardport/tests/test_updates.py b/forwardport/tests/test_updates.py
index 0dedf9dc..73730d86 100644
--- a/forwardport/tests/test_updates.py
+++ b/forwardport/tests/test_updates.py
@@ -470,10 +470,16 @@ CONFLICT (add/add): Merge conflict in h
 
 stderr:
 ```
-$$:$$:$$.$$ {pr2_id.head}
-error: could not apply $$... newfiles
-hint: $$
+$$ trace: built-in: git cherry-pick {pr2_id.head} --strategy ort
+error: could not apply {pr2_id.head[:7]}... newfiles
+hint: After resolving the conflicts, mark them with
+hint: "git add/rm <pathspec>", then run
+hint: "git cherry-pick --continue".
+hint: You can instead skip this commit with "git cherry-pick --skip".
+hint: To abort and get back to the state before "git cherry-pick",
+hint: run "git cherry-pick --abort".
+hint: Disable this message with "git config advice.mergeConflict false"
 ----------
 status:
-'''))
+```'''))
     ]

From fec3d39d1965d32403f9072242e1bb163e6ec733 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 6 Jun 2024 11:07:57 +0200
Subject: [PATCH 113/192] [ADD] *: per-repository webhook secret

Currently webhook secrets are configured per *project* which is an
issue both because different repositories may have different
administrators and thus creates safety concerns, and because multiple
repositories can feed into different projects (e.g. on mergebot,
odoo-dev/odoo is both an ancillary repository to the main RD project,
and the main repository to the minor / legacy master-wowl
project). This means it can be necessary to have multiple projects
share the same secret as well, this then mandates the secret for more
repositories per (1).

This is a pain in the ass, so just detach secrets from projects and
link them *only* to repositories, it's cleaner and easier to manage
and set up progressively.

This requires a lot of changes to the tests, as they all need to
correctly configure the signaling.

For `runbot_merge` there was *some* setup sharing already via the
module-level `repo` fixtures`, those were merged into a conftest-level
fixture which could handle the signaling setup. A few tests which
unnecessarily set up repositories ad-hoc were also moved to the
fixture. But for most of the ad-hoc setup in `runbot_merge`, as well
as `forwardport` where it's all ad-hoc, events sources setup was just
appended as is. This should probably be cleaned up at one point, with
the various requirements collected and organised into a small set of
fixtures doing the job more uniformly.

Fixes #887
---
 conftest.py                                   |  4 ++
 forwardport/tests/test_limit.py               |  1 +
 forwardport/tests/test_simple.py              |  1 -
 forwardport/tests/test_updates.py             |  1 +
 forwardport/tests/test_weird.py               |  3 ++
 mergebot_test_utils/utils.py                  |  1 +
 runbot_merge/__manifest__.py                  |  2 +-
 runbot_merge/controllers/__init__.py          | 25 ++++++----
 .../migrations/15.0.1.14/pre-migration.py     | 12 +++++
 runbot_merge/models/__init__.py               |  1 +
 runbot_merge/models/events_sources.py         | 12 +++++
 runbot_merge/models/project.py                |  6 ---
 runbot_merge/security/ir.model.access.csv     |  2 +
 runbot_merge/tests/conftest.py                | 27 ++++++++++
 runbot_merge/tests/test_basic.py              | 26 ++++------
 runbot_merge/tests/test_batch_consistency.py  | 29 ++---------
 runbot_merge/tests/test_by_branch.py          | 35 ++++++-------
 runbot_merge/tests/test_disabled_branch.py    |  3 ++
 runbot_merge/tests/test_multirepo.py          | 21 +++++---
 runbot_merge/tests/test_oddities.py           | 50 ++++---------------
 runbot_merge/tests/test_project_toggles.py    | 11 ----
 runbot_merge/tests/test_staging.py            | 13 -----
 runbot_merge/tests/test_status_overrides.py   |  3 ++
 runbot_merge/views/configuration.xml          | 17 +++++++
 runbot_merge/views/runbot_merge_project.xml   |  1 -
 25 files changed, 154 insertions(+), 153 deletions(-)
 create mode 100644 runbot_merge/migrations/15.0.1.14/pre-migration.py
 create mode 100644 runbot_merge/models/events_sources.py

diff --git a/conftest.py b/conftest.py
index db3822f7..946c9466 100644
--- a/conftest.py
+++ b/conftest.py
@@ -452,6 +452,10 @@ def check(response):
 # to) break the existing local tests
 @pytest.fixture
 def make_repo(capsys, request, config, tunnel, users):
+    """Fixtures which creates a repository on the github side, plugs webhooks
+    in, and registers the repository for deletion on cleanup (unless
+    ``--no-delete`` is set)
+    """
     owner = config['github']['owner']
     github = requests.Session()
     github.headers['Authorization'] = 'token %s' % config['github']['token']
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index 071d2a54..673a2bcf 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -421,6 +421,7 @@ def post_merge(env, config, users, make_repo, branches):
             })
         ]
     })
+    env['runbot_merge.events_sources'].create({'repository': prod.name})
 
     env['res.partner'].search([
         ('github_login', '=', config['role_reviewer']['user'])
diff --git a/forwardport/tests/test_simple.py b/forwardport/tests/test_simple.py
index 901b93a7..eae53ce4 100644
--- a/forwardport/tests/test_simple.py
+++ b/forwardport/tests/test_simple.py
@@ -35,7 +35,6 @@ def test_straightforward_flow(env, config, make_repo, users):
     other_user = config['role_other']
     other_user_repo = prod.fork(token=other_user['token'])
 
-    project = env['runbot_merge.project'].search([])
     b_head = prod.commit('b')
     c_head = prod.commit('c')
     with prod, other_user_repo:
diff --git a/forwardport/tests/test_updates.py b/forwardport/tests/test_updates.py
index 73730d86..0ee884d2 100644
--- a/forwardport/tests/test_updates.py
+++ b/forwardport/tests/test_updates.py
@@ -326,6 +326,7 @@ def test_duplicate_fw(env, make_repo, setreviewers, config, users):
         ]
     })
     setreviewers(*proj.repo_ids)
+    env['runbot_merge.events_sources'].create({'repository': repo.name})
 
     # create a PR in v1, merge it, then create all 3 ports
     with repo:
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index 74862760..00dd86be 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -206,6 +206,7 @@ class TestNotAllBranches:
             'branch_filter': '[("name", "in", ["a", "c"])]',
         })
         setreviewers(repo_a, repo_b)
+        env['runbot_merge.events_sources'].create([{'repository': a.name}, {'repository': b.name}])
         return project, a, a_dev, b, b_dev
 
     def test_single_first(self, env, repos, config):
@@ -883,6 +884,7 @@ def test_disable_branch_with_batches(env, config, make_repo, users):
         "required_statuses": "default",
         "fp_remote_target": fork2.name,
     })
+    env['runbot_merge.events_sources'].create({'repository': repo2.name})
     env['res.partner'].search([
         ('github_login', '=', config['role_reviewer']['user'])
     ]).write({
@@ -978,6 +980,7 @@ def test_disable_multitudes(env, config, make_repo, users, setreviewers):
         })],
     })
     setreviewers(project.repo_ids)
+    env['runbot_merge.events_sources'].create({'repository': repo.name})
 
     with repo:
         [a, b, c, d] = repo.make_commits(
diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index 0d0f7dff..bc451293 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -130,6 +130,7 @@ def make_basic(
         })
 
     prod = make_repo(reponame)
+    env['runbot_merge.events_sources'].create({'repository': prod.name})
     with prod:
         a_0, a_1, a_2, a_3, a_4, = prod.make_commits(
             None,
diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py
index b005c5a9..5f23262c 100644
--- a/runbot_merge/__manifest__.py
+++ b/runbot_merge/__manifest__.py
@@ -1,6 +1,6 @@
 {
     'name': 'merge bot',
-    'version': '1.13',
+    'version': '1.14',
     'depends': ['contacts', 'mail', 'website'],
     'data': [
         'security/security.xml',
diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 5485f1fa..bdbd96c8 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -86,18 +86,18 @@ class MergebotController(Controller):
 
         github._gh.info(self._format(req))
 
-        c = EVENTS.get(event)
-        if not c:
-            _logger.warning('Unknown event %s', event)
-            return 'Unknown event {}'.format(event)
-
-        repo = request.jsonrequest['repository']['full_name']
+        repo = request.jsonrequest.get('repository', {}).get('full_name')
         env = request.env(user=1)
 
-        secret = env['runbot_merge.repository'].search([
-            ('name', '=', repo),
-        ]).project_id.secret
-        if secret and secret.strip():
+        source = repo and env['runbot_merge.events_sources'].search([('repository', '=', repo)])
+        if not source:
+            _logger.warning(
+                "Ignored hook %s to unknown source repository %s",
+                req.headers.get("X-Github-Delivery"),
+                repo,
+            )
+            return werkzeug.exceptions.Forbidden()
+        elif secret := source.secret:
             signature = 'sha256=' + hmac.new(secret.strip().encode(), req.get_data(), hashlib.sha256).hexdigest()
             if not hmac.compare_digest(signature, req.headers.get('X-Hub-Signature-256', '')):
                 _logger.warning(
@@ -114,6 +114,11 @@ class MergebotController(Controller):
         else:
             _logger.info("No secret or signature for %s", repo)
 
+        c = EVENTS.get(event)
+        if not c:
+            _logger.warning('Unknown event %s', event)
+            return 'Unknown event {}'.format(event)
+
         sentry_sdk.set_context('webhook', request.jsonrequest)
         return c(env, request.jsonrequest)
 
diff --git a/runbot_merge/migrations/15.0.1.14/pre-migration.py b/runbot_merge/migrations/15.0.1.14/pre-migration.py
new file mode 100644
index 00000000..6ac9ed1f
--- /dev/null
+++ b/runbot_merge/migrations/15.0.1.14/pre-migration.py
@@ -0,0 +1,12 @@
+def migrate(cr, version):
+    cr.execute("""
+    CREATE TABLE runbot_merge_events_sources (
+        id serial primary key,
+        repository varchar not null,
+        secret varchar
+    );
+    INSERT INTO runbot_merge_events_sources (repository, secret)
+    SELECT r.name, p.secret
+    FROM runbot_merge_repository r
+    JOIN runbot_merge_project p ON p.id = r.project_id;
+    """)
diff --git a/runbot_merge/models/__init__.py b/runbot_merge/models/__init__.py
index 18367537..f15f98f2 100644
--- a/runbot_merge/models/__init__.py
+++ b/runbot_merge/models/__init__.py
@@ -6,4 +6,5 @@ from . import batch
 from . import project_freeze
 from . import stagings_create
 from . import staging_cancel
+from . import events_sources
 from . import crons
diff --git a/runbot_merge/models/events_sources.py b/runbot_merge/models/events_sources.py
new file mode 100644
index 00000000..34db75ac
--- /dev/null
+++ b/runbot_merge/models/events_sources.py
@@ -0,0 +1,12 @@
+from odoo import models, fields
+
+
+class EventsSources(models.Model):
+    _name = 'runbot_merge.events_sources'
+    _description = 'Valid Webhook Event Sources'
+    _order = "repository"
+    _rec_name = "repository"
+
+    # FIXME: unique repo? Or allow multiple secrets per repo?
+    repository = fields.Char(index=True, required=True)
+    secret = fields.Char()
diff --git a/runbot_merge/models/project.py b/runbot_merge/models/project.py
index db41758b..55210ebb 100644
--- a/runbot_merge/models/project.py
+++ b/runbot_merge/models/project.py
@@ -55,12 +55,6 @@ class Project(models.Model):
     batch_limit = fields.Integer(
         default=8, group_operator=None, help="Maximum number of PRs staged together")
 
-    secret = fields.Char(
-        help="Webhook secret. If set, will be checked against the signature "
-             "of (valid) incoming webhook signatures, failing signatures "
-             "will lead to webhook rejection. Should only use ASCII."
-    )
-
     freeze_id = fields.Many2one('runbot_merge.project.freeze', compute='_compute_freeze')
     freeze_reminder = fields.Text()
 
diff --git a/runbot_merge/security/ir.model.access.csv b/runbot_merge/security/ir.model.access.csv
index 28f2bd7e..9d2da846 100644
--- a/runbot_merge/security/ir.model.access.csv
+++ b/runbot_merge/security/ir.model.access.csv
@@ -21,6 +21,7 @@ access_runbot_merge_fetch_job_admin,Admin access to fetch jobs,model_runbot_merg
 access_runbot_merge_pull_requests_feedback_admin,Admin access to feedback,model_runbot_merge_pull_requests_feedback,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_review_rights,Admin access to review permissions,model_res_partner_review,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_review_override,Admin access to override permissions,model_res_partner_override,runbot_merge.group_admin,1,1,1,1
+access_runbot_merge_events_sources,Admin access to event sources,model_runbot_merge_events_sources,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_project,User access to project,model_runbot_merge_project,base.group_user,1,0,0,0
 access_runbot_merge_repository,User access to repo,model_runbot_merge_repository,base.group_user,1,0,0,0
 access_runbot_merge_branch,User access to branches,model_runbot_merge_branch,base.group_user,1,0,0,0
@@ -29,3 +30,4 @@ access_runbot_merge_pull_requests_feedback,Users have no reason to access feedba
 access_runbot_merge_review_rights_2,Users can see partners,model_res_partner_review,base.group_user,1,0,0,0
 access_runbot_merge_review_override_2,Users can see partners,model_res_partner_override,base.group_user,1,0,0,0
 runbot_merge.access_runbot_merge_pull_requests_feedback_template,access_runbot_merge_pull_requests_feedback_template,runbot_merge.model_runbot_merge_pull_requests_feedback_template,base.group_system,1,1,0,0
+
diff --git a/runbot_merge/tests/conftest.py b/runbot_merge/tests/conftest.py
index 75cad7a8..f75447ca 100644
--- a/runbot_merge/tests/conftest.py
+++ b/runbot_merge/tests/conftest.py
@@ -39,3 +39,30 @@ def project(env, config):
         'github_prefix': 'hansen',
         'branch_ids': [(0, 0, {'name': 'master'})],
     })
+
+
+@pytest.fixture
+def make_repo2(env, project, make_repo, users, setreviewers):
+    """Layer over ``make_repo`` which also:
+
+    - adds the new repo to ``project`` (with no group and the ``'default'`` status required)
+    - sets the standard reviewers on the repo
+    - and creates an event source for the repo
+    """
+    def mr(name):
+        r = make_repo(name)
+        rr = env['runbot_merge.repository'].create({
+            'project_id': project.id,
+            'name': r.name,
+            'group_id': False,
+            'required_statuses': 'default',
+        })
+        setreviewers(rr)
+        env['runbot_merge.events_sources'].create({'repository': r.name})
+        return r
+    return mr
+
+
+@pytest.fixture
+def repo(make_repo2):
+    return make_repo2('repo')
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index d43368e9..125d1477 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -12,17 +12,9 @@ from lxml import html
 import odoo
 from utils import _simple_init, seen, matches, get_partner, Commit, pr_page, to_pr, part_of, ensure_one
 
-
-@pytest.fixture
-def repo(env, project, make_repo, users, setreviewers):
-    r = make_repo('repo')
-    project.write({'repo_ids': [(0, 0, {
-        'name': r.name,
-        'group_id': False,
-        'required_statuses': 'legal/cla,ci/runbot'
-    })]})
-    setreviewers(*project.repo_ids)
-    return r
+@pytest.fixture(autouse=True)
+def _configure_statuses(project, repo):
+    project.repo_ids.required_statuses = 'legal/cla,ci/runbot'
 
 @pytest.fixture(autouse=True, params=["statuses", "rpc"])
 def stagings(request, env, project, repo):
@@ -353,11 +345,15 @@ Co-authored-by: Bob <bob@example.com>""".format(
         )
 
 class TestWebhookSecurity:
+    @pytest.fixture(autouse=True)
+    def add_secret_to_source(self, env, repo):
+        env['runbot_merge.events_sources'].search([
+            ('repository', '=', repo.name),
+        ]).secret = "a secret"
+
     def test_no_secret(self, env, project, repo):
         """ Test 1: didn't add a secret to the repo, should be ignored
         """
-        project.secret = "a secret"
-
         with repo:
             m = repo.make_commit(None, "initial", None, tree={'a': 'some content'})
             repo.make_ref('heads/master', m)
@@ -371,7 +367,6 @@ class TestWebhookSecurity:
         ])
 
     def test_wrong_secret(self, env, project, repo):
-        project.secret = "a secret"
         with repo:
             repo.set_secret("wrong secret")
 
@@ -387,7 +382,6 @@ class TestWebhookSecurity:
         ])
 
     def test_correct_secret(self, env, project, repo):
-        project.secret = "a secret"
         with repo:
             repo.set_secret("a secret")
 
@@ -2789,7 +2783,7 @@ class TestBatching(object):
     def test_batching_pressing(self, env, repo, config):
         """ "Pressing" PRs should be selected before normal & batched together
         """
-        # by limiting the batch size to 3 we allow both high-priority PRs, but 
+        # by limiting the batch size to 3 we allow both high-priority PRs, but
         # a single normal priority one
         env['runbot_merge.project'].search([]).batch_limit = 3
         with repo:
diff --git a/runbot_merge/tests/test_batch_consistency.py b/runbot_merge/tests/test_batch_consistency.py
index ef3c88f6..4a35389b 100644
--- a/runbot_merge/tests/test_batch_consistency.py
+++ b/runbot_merge/tests/test_batch_consistency.py
@@ -4,19 +4,10 @@ without wider relevance and thus other location.
 from utils import Commit, to_pr
 
 
-def test_close_single(env, project, make_repo, setreviewers):
+def test_close_single(env, repo):
     """If a batch has a single PR and that PR gets closed, the batch should be
     inactive *and* blocked.
     """
-    repo = make_repo('wheee')
-    r = env['runbot_merge.repository'].create({
-        'project_id': project.id,
-        'name': repo.name,
-        'required_statuses': 'default',
-        'group_id': False,
-    })
-    setreviewers(r)
-
     with repo:
         repo.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
         [c] = repo.make_commits('master', Commit('b', tree={"b": "b"}))
@@ -41,25 +32,13 @@ def test_close_single(env, project, make_repo, setreviewers):
 
     assert Batches.search_count([]) == 0
 
-def test_close_multiple(env, project, make_repo, setreviewers):
+def test_close_multiple(env, make_repo2):
     """If a batch has a single PR and that PR gets closed, the batch should be
     inactive *and* blocked.
     """
     Batches = env['runbot_merge.batch']
-    repo1 = make_repo('wheee')
-    repo2 = make_repo('wheeee')
-    project.write({
-        'repo_ids': [(0, 0, {
-            'name': repo1.name,
-            'required_statuses': 'default',
-            'group_id': False,
-        }), (0, 0, {
-            'name': repo2.name,
-            'required_statuses': 'default',
-            'group_id': False,
-        })]
-    })
-    setreviewers(*project.repo_ids)
+    repo1 = make_repo2('wheee')
+    repo2 = make_repo2('wheeee')
 
     with repo1:
         repo1.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
diff --git a/runbot_merge/tests/test_by_branch.py b/runbot_merge/tests/test_by_branch.py
index 4122bf82..efaf0394 100644
--- a/runbot_merge/tests/test_by_branch.py
+++ b/runbot_merge/tests/test_by_branch.py
@@ -2,28 +2,21 @@ import pytest
 
 from utils import Commit
 
-
 @pytest.fixture
-def repo(env, project, make_repo, users, setreviewers):
-    r = make_repo('repo')
-    project.write({
-        'repo_ids': [(0, 0, {
-            'name': r.name,
-            'status_ids': [
-                (0, 0, {'context': 'ci'}),
-                # require the lint status on master
-                (0, 0, {
-                    'context': 'lint',
-                    'branch_filter': [('id', '=', project.branch_ids.id)]
-                }),
-                (0, 0, {'context': 'pr', 'stagings': False}),
-                (0, 0, {'context': 'staging', 'prs': False}),
-            ]
-        })],
-    })
-    setreviewers(*project.repo_ids)
-    return r
+def _setup_statuses(project, repo):
+    project.repo_ids.status_ids = [
+        (5, 0, 0),
+        (0, 0, {'context': 'ci'}),
+        # require the lint status on master
+        (0, 0, {
+            'context': 'lint',
+            'branch_filter': [('id', '=', project.branch_ids.id)]
+        }),
+        (0, 0, {'context': 'pr', 'stagings': False}),
+        (0, 0, {'context': 'staging', 'prs': False}),
+    ]
 
+@pytest.mark.usefixtures('_setup_statuses')
 def test_status_applies(env, repo, config):
     """ If branches are associated with a repo status, only those branch should
     require the status on their PRs & stagings
@@ -71,6 +64,7 @@ def test_status_applies(env, repo, config):
     env.run_crons('runbot_merge.process_updated_commits')
     assert st.state == 'success'
 
+@pytest.mark.usefixtures('_setup_statuses')
 def test_status_skipped(env, project, repo, config):
     """ Branches not associated with a repo status should not require the status
     on their PRs or stagings
@@ -132,6 +126,7 @@ def test_pseudo_version_tag(env, project, make_repo, setreviewers, config):
         ],
     })
     setreviewers(*project.repo_ids)
+    env['runbot_merge.events_sources'].create({'repository': repo.name})
 
     with repo:
         [m] = repo.make_commits(None, Commit('c1', tree={'a': '1'}), ref='heads/master')
diff --git a/runbot_merge/tests/test_disabled_branch.py b/runbot_merge/tests/test_disabled_branch.py
index ed970974..cfd16bae 100644
--- a/runbot_merge/tests/test_disabled_branch.py
+++ b/runbot_merge/tests/test_disabled_branch.py
@@ -23,6 +23,7 @@ def test_existing_pr_disabled_branch(env, project, make_repo, setreviewers, conf
         'group_id': False,
     })
     setreviewers(*project.repo_ids)
+    env['runbot_merge.events_sources'].create({'repository': repo.name})
 
     with repo:
         [m] = repo.make_commits(None, Commit('root', tree={'a': '1'}), ref='heads/master')
@@ -107,6 +108,7 @@ def test_new_pr_no_branch(env, project, make_repo, setreviewers, users):
         'status_ids': [(0, 0, {'context': 'status'})]
     })
     setreviewers(*project.repo_ids)
+    env['runbot_merge.events_sources'].create({'repository': repo.name})
 
     with repo:
         [m] = repo.make_commits(None, Commit('root', tree={'a': '1'}), ref='heads/master')
@@ -140,6 +142,7 @@ def test_new_pr_disabled_branch(env, project, make_repo, setreviewers, users):
         'active': False,
     })
     setreviewers(*project.repo_ids)
+    env['runbot_merge.events_sources'].create({'repository': repo.name})
 
     with repo:
         [m] = repo.make_commits(None, Commit('root', tree={'a': '1'}), ref='heads/master')
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 42043f78..e6073028 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -19,39 +19,42 @@ from utils import seen, get_partner, pr_page, to_pr, Commit
 
 
 @pytest.fixture
-def repo_a(project, make_repo, setreviewers):
+def repo_a(env, project, make_repo, setreviewers):
     repo = make_repo('a')
-    r = project.env['runbot_merge.repository'].create({
+    r = env['runbot_merge.repository'].create({
         'project_id': project.id,
         'name': repo.name,
         'required_statuses': 'default',
         'group_id': False,
     })
     setreviewers(r)
+    env['runbot_merge.events_sources'].create({'repository': r.name})
     return repo
 
 @pytest.fixture
-def repo_b(project, make_repo, setreviewers):
+def repo_b(env, project, make_repo, setreviewers):
     repo = make_repo('b')
-    r = project.env['runbot_merge.repository'].create({
+    r = env['runbot_merge.repository'].create({
         'project_id': project.id,
         'name': repo.name,
         'required_statuses': 'default',
         'group_id': False,
     })
     setreviewers(r)
+    env['runbot_merge.events_sources'].create({'repository': r.name})
     return repo
 
 @pytest.fixture
-def repo_c(project, make_repo, setreviewers):
+def repo_c(env, project, make_repo, setreviewers):
     repo = make_repo('c')
-    r = project.env['runbot_merge.repository'].create({
+    r = env['runbot_merge.repository'].create({
         'project_id': project.id,
         'name': repo.name,
         'required_statuses': 'default',
         'group_id': False,
     })
     setreviewers(r)
+    env['runbot_merge.events_sources'].create({'repository': r.name})
     return repo
 
 def make_pr(repo, prefix, trees, *, target='master', user,
@@ -934,6 +937,7 @@ class TestSubstitutions:
             'repo_ids': [(0, 0, {'name': 'xxx/xxx'})],
             'branch_ids': [(0, 0, {'name': 'master'})]
         })
+        env['runbot_merge.events_sources'].create({'repository': 'xxx/xxx'})
         r = p.repo_ids
         # replacement pattern, pr label, stored label
         cases = [
@@ -1001,8 +1005,7 @@ class TestSubstitutions:
         with repo_a:
             repo_a.make_commits('master', repo_a.Commit('bop', tree={'a': '1'}), ref='heads/abranch')
             pra = repo_a.make_pr(target='master', head='abranch')
-        b_fork = repo_b.fork()
-        with b_fork, repo_b:
+        with repo_b, repo_b.fork() as b_fork:
             b_fork.make_commits('master', b_fork.Commit('pob', tree={'b': '1'}), ref='heads/abranch')
             prb = repo_b.make_pr(
                 title="a pr",
@@ -1058,6 +1061,7 @@ def test_multi_project(env, make_repo, setreviewers, users, config,
         'branch_ids': [(0, 0, {'name': 'default'})],
     })
     setreviewers(*p1.repo_ids)
+    env['runbot_merge.events_sources'].create([{'repository': r1.name}])
 
     r2 = make_repo('repo_b')
     with r2:
@@ -1078,6 +1082,7 @@ def test_multi_project(env, make_repo, setreviewers, users, config,
         'branch_ids': [(0, 0, {'name': 'default'})],
     })
     setreviewers(*p2.repo_ids)
+    env['runbot_merge.events_sources'].create([{'repository': r2.name}])
 
     assert r1_dev.owner == r2_dev.owner
 
diff --git a/runbot_merge/tests/test_oddities.py b/runbot_merge/tests/test_oddities.py
index b5d9d5b2..8b2bba21 100644
--- a/runbot_merge/tests/test_oddities.py
+++ b/runbot_merge/tests/test_oddities.py
@@ -96,7 +96,7 @@ def test_unreviewer(env, project, port):
 
     assert p.review_rights == env['res.partner.review']
 
-def test_staging_post_update(env, project, make_repo, setreviewers, users, config):
+def test_staging_post_update(env, repo, users, config):
     """Because statuses come from commits, it's possible to update the commits
     of a staging after that staging has completed (one way or the other), either
     by sending statuses directly (e.g. rebuilding, for non-deterministic errors)
@@ -105,21 +105,13 @@ def test_staging_post_update(env, project, make_repo, setreviewers, users, confi
     This makes post-mortem analysis quite confusing, so stagings should
     "lock in" their statuses once they complete.
     """
-    repo = make_repo('repo')
-    project.write({'repo_ids': [(0, 0, {
-        'name': repo.name,
-        'group_id': False,
-        'required_statuses': 'legal/cla,ci/runbot'
-    })]})
-    setreviewers(*project.repo_ids)
 
     with repo:
         [m] = repo.make_commits(None, Commit('initial', tree={'m': 'm'}), ref='heads/master')
 
         repo.make_commits(m, Commit('thing', tree={'m': 'c'}), ref='heads/other')
         pr = repo.make_pr(target='master', head='other')
-        repo.post_status(pr.head, 'success', 'ci/runbot')
-        repo.post_status(pr.head, 'success', 'legal/cla')
+        repo.post_status(pr.head, 'success')
         pr.post_comment('hansen r+ rebase-merge', config['role_reviewer']['token'])
     env.run_crons()
     pr_id = to_pr(env, pr)
@@ -128,33 +120,25 @@ def test_staging_post_update(env, project, make_repo, setreviewers, users, confi
 
     staging_head = repo.commit('staging.master')
     with repo:
-        repo.post_status(staging_head, 'failure', 'ci/runbot')
+        repo.post_status(staging_head, 'failure')
     env.run_crons()
     assert pr_id.state == 'error'
     assert staging_id.state == 'failure'
     assert staging_id.statuses == [
-        [repo.name, 'ci/runbot', 'failure', ''],
+        [repo.name, 'default', 'failure', ''],
     ]
 
     with repo:
-        repo.post_status(staging_head, 'success', 'ci/runbot')
+        repo.post_status(staging_head, 'success')
     env.run_crons()
     assert staging_id.state == 'failure'
     assert staging_id.statuses == [
-        [repo.name, 'ci/runbot', 'failure', ''],
+        [repo.name, 'default', 'failure', ''],
     ]
 
-def test_merge_empty_commits(env, project, make_repo, setreviewers, users, config):
+def test_merge_empty_commits(env, repo, users, config):
     """The mergebot should allow merging already-empty commits.
     """
-    repo = make_repo('repo')
-    project.write({'repo_ids': [(0, 0, {
-        'name': repo.name,
-        'group_id': False,
-        'required_statuses': 'default',
-    })]})
-    setreviewers(*project.repo_ids)
-
     with repo:
         [m] = repo.make_commits(None, Commit('initial', tree={'m': 'm'}), ref='heads/master')
 
@@ -187,18 +171,10 @@ def test_merge_empty_commits(env, project, make_repo, setreviewers, users, confi
     assert commits[1]['commit']['message'].startswith('thing1')
     assert commits[2]['commit']['message'] == 'initial'
 
-def test_merge_emptying_commits(env, project, make_repo, setreviewers, users, config):
+def test_merge_emptying_commits(env, repo, users, config):
     """The mergebot should *not* allow merging non-empty commits which become
     empty as part of the staging (rebasing)
     """
-    repo = make_repo('repo')
-    project.write({'repo_ids': [(0, 0, {
-        'name': repo.name,
-        'group_id': False,
-        'required_statuses': 'default',
-    })]})
-    setreviewers(*project.repo_ids)
-
     with repo:
         [m, _] = repo.make_commits(
             None,
@@ -253,15 +229,7 @@ def test_merge_emptying_commits(env, project, make_repo, setreviewers, users, co
         (users['user'], f"{ping} unable to stage: results in an empty tree when merged, might be the duplicate of a merged PR.")
     ]
 
-def test_force_ready(env, make_repo, project, setreviewers, config):
-    repo = make_repo('repo')
-    project.write({'repo_ids': [(0, 0, {
-        'name': repo.name,
-        'group_id': False,
-        'required_statuses': 'default',
-    })]})
-    setreviewers(*project.repo_ids)
-
+def test_force_ready(env, repo, config):
     with repo:
         [m] = repo.make_commits(None, Commit('initial', tree={'m': 'm'}), ref="heads/master")
 
diff --git a/runbot_merge/tests/test_project_toggles.py b/runbot_merge/tests/test_project_toggles.py
index 876c241e..615e6e4b 100644
--- a/runbot_merge/tests/test_project_toggles.py
+++ b/runbot_merge/tests/test_project_toggles.py
@@ -6,17 +6,6 @@ import pytest
 from utils import Commit, to_pr, ensure_one
 
 
-@pytest.fixture
-def repo(env, project, make_repo, users, setreviewers):
-    r = make_repo('repo')
-    project.write({'repo_ids': [(0, 0, {
-        'name': r.name,
-        'group_id': False,
-        'required_statuses': 'default',
-    })]})
-    setreviewers(*project.repo_ids)
-    return r
-
 def test_disable_staging(env, project, repo, config):
     """In order to avoid issues of cron locking, as well as not disable staging
     for every project when trying to freeze just one of them (cough cough), a
diff --git a/runbot_merge/tests/test_staging.py b/runbot_merge/tests/test_staging.py
index 8903df62..c1aeeeeb 100644
--- a/runbot_merge/tests/test_staging.py
+++ b/runbot_merge/tests/test_staging.py
@@ -1,19 +1,6 @@
-import pytest
-
 from utils import Commit, to_pr
 
 
-@pytest.fixture
-def repo(env, project, make_repo, users, setreviewers):
-    r = make_repo('repo')
-    project.write({'repo_ids': [(0, 0, {
-        'name': r.name,
-        'group_id': False,
-        'required_statuses': 'default'
-    })]})
-    setreviewers(*project.repo_ids)
-    return r
-
 def test_staging_disabled_branch(env, project, repo, config):
     """Check that it's possible to disable staging on a specific branch
     """
diff --git a/runbot_merge/tests/test_status_overrides.py b/runbot_merge/tests/test_status_overrides.py
index 1bdc4877..0d13ac61 100644
--- a/runbot_merge/tests/test_status_overrides.py
+++ b/runbot_merge/tests/test_status_overrides.py
@@ -50,6 +50,7 @@ def test_basic(env, project, make_repo, users, setreviewers, config):
         'status_ids': [(0, 0, {'context': 'l/int'})]
     })
     setreviewers(*project.repo_ids)
+    env['runbot_merge.events_sources'].create({'repository': repo.name})
     # "other" can override the lint
     env['res.partner'].create({
         'name': config['role_other'].get('name', 'Other'),
@@ -110,6 +111,7 @@ def test_multiple(env, project, make_repo, users, setreviewers, config):
         'status_ids': [(0, 0, {'context': 'l/int'}), (0, 0, {'context': 'c/i'})]
     })
     setreviewers(*project.repo_ids)
+    env['runbot_merge.events_sources'].create({'repository': repo.name})
     # "other" can override the lints
     env['res.partner'].create({
         'name': config['role_other'].get('name', 'Other'),
@@ -174,6 +176,7 @@ def test_no_repository(env, project, make_repo, users, setreviewers, config):
         'status_ids': [(0, 0, {'context': 'l/int'})]
     })
     setreviewers(*project.repo_ids)
+    env['runbot_merge.events_sources'].create({'repository': repo.name})
     # "other" can override the lint
     env['res.partner'].create({
         'name': config['role_other'].get('name', 'Other'),
diff --git a/runbot_merge/views/configuration.xml b/runbot_merge/views/configuration.xml
index 32d3f3e7..0273a53a 100644
--- a/runbot_merge/views/configuration.xml
+++ b/runbot_merge/views/configuration.xml
@@ -63,6 +63,21 @@
         </field>
     </record>
 
+    <record id="action_events_sources" model="ir.actions.act_window">
+        <field name="name">Events Sources</field>
+        <field name="res_model">runbot_merge.events_sources</field>
+    </record>
+    <record id="tree_events_sources" model="ir.ui.view">
+        <field name="name">Events Sources List</field>
+        <field name="model">runbot_merge.events_sources</field>
+        <field name="arch" type="xml">
+            <tree editable="bottom">
+                <field name="repository"/>
+                <field name="secret"/>
+            </tree>
+        </field>
+    </record>
+
     <menuitem name="Configuration" id="menu_configuration" parent="runbot_merge_menu">
         <menuitem name="CI Overrides" id="menu_configuration_overrides"
                   action="action_overrides"/>
@@ -70,5 +85,7 @@
                   action="action_review"/>
         <menuitem name="Feedback Templates" id="menu_configuration_feedback"
                   action="action_feedback"/>
+        <menuitem name="Events Sources" id="menu_configuration_events_sources"
+                  action="action_events_sources"/>
     </menuitem>
 </odoo>
diff --git a/runbot_merge/views/runbot_merge_project.xml b/runbot_merge/views/runbot_merge_project.xml
index 68eb28cc..6e83376f 100644
--- a/runbot_merge/views/runbot_merge_project.xml
+++ b/runbot_merge/views/runbot_merge_project.xml
@@ -29,7 +29,6 @@
                                    help="Identity when creating new commits, defaults to github name, falls back to login."/>
                             <field name="github_email" readonly="0"
                                    help="Identity when creating new commits, defaults to public email, falls back to primary email."/>
-                            <field name="secret"/>
                             <span attrs="{'invisible': [
                                 '|',
                                     ('staging_statuses', '=', False),

From ceb2bd457ed40b838ab70a839a41c8cbcc6acab2 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 7 Jun 2024 15:02:28 +0200
Subject: [PATCH 114/192] [IMP] forwardport: automatically approve freeze
 backfills

if the corresponding master PR was approved. That the backfill needs
approval can be considered a race condition: its dual is approved
and *might* have been merged before the freeze, but was not, so a
backfill was needed.

Copying over the approval is a convenience feature with pretty much no
actual risk I can think of.

Fixes #884
---
 forwardport/models/forwardport.py | 40 +++++++++++++-------
 forwardport/tests/test_weird.py   | 62 ++++++++++++++++++++++++++++++-
 2 files changed, 86 insertions(+), 16 deletions(-)

diff --git a/forwardport/models/forwardport.py b/forwardport/models/forwardport.py
index 75823461..54521621 100644
--- a/forwardport/models/forwardport.py
+++ b/forwardport/models/forwardport.py
@@ -108,20 +108,32 @@ class ForwardPortTasks(models.Model, Queue):
         )
         # insert new batch in ancestry sequence
         if self.source == 'insert':
-            self.env['runbot_merge.batch'].search([
-                ('parent_id', '=', batch.id),
-                ('id', '!=', newbatch.id),
-            ]).parent_id = newbatch.id
-            # insert new PRs in ancestry sequence unless conflict (= no parent)
-            for pr in newbatch.prs:
-                if not pr.parent_id:
-                    break
-                newchild = pr.search([
-                    ('parent_id', '=', pr.parent_id.id),
-                    ('id', '!=', pr.id),
-                ])
-                if newchild:
-                    newchild.parent_id = pr.id
+            self._process_insert(batch, newbatch)
+
+    def _process_insert(self, batch, newbatch):
+        self.env['runbot_merge.batch'].search([
+            ('parent_id', '=', batch.id),
+            ('id', '!=', newbatch.id),
+        ]).parent_id = newbatch.id
+        # insert new PRs in ancestry sequence unless conflict (= no parent)
+        for pr in newbatch.prs:
+            next_target = pr._find_next_target()
+            if not next_target:
+                continue
+
+            # should have one since it was inserted before an other PR?
+            descendant = pr.search([
+                ('target', '=', next_target.id),
+                ('source_id', '=', pr.source_id.id),
+            ])
+
+            # copy the reviewing of the "descendant" (even if detached) to this pr
+            if reviewer := descendant.reviewed_by:
+                pr.reviewed_by = reviewer
+
+            # replace parent_id *if not detached*
+            if descendant.parent_id:
+                descendant.parent_id = pr.id
 
     def _complete_batches(self):
         source = pr = self.pr_id
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index 00dd86be..e0922ceb 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -750,9 +750,14 @@ def test_freeze(env, config, make_repo, users):
     """Freeze:
 
     - should not forward-port the freeze PRs themselves
+    - unmerged forward ports need to be backfilled
+    - if the tip of the forward port is approved, the backfilled forward port
+      should also be
     """
-    prod, _ = make_basic(env, config, make_repo)
+    prod, _ = make_basic(env, config, make_repo, statuses='default')
     project = env['runbot_merge.project'].search([])
+
+
     # branches here are "a" (older), "b", and "c" (master)
     with prod:
         [root, _] = prod.make_commits(
@@ -762,6 +767,22 @@ def test_freeze(env, config, make_repo, users):
             ref='heads/b'
         )
         prod.make_commits(root, Commit('other', tree={'f': '1'}), ref='heads/c')
+
+    # region PR which is forward ported but the FPs are not merged (they are approved)
+    with prod:
+        prod.make_commits("a", Commit("stuff", tree={'x': '0'}), ref="heads/abranch")
+        p = prod.make_pr(target='a', head='abranch')
+        p.post_comment("hansen r+ fw=skipci", config['role_reviewer']['token'])
+        prod.post_status('abranch', 'success')
+    env.run_crons()
+    with prod:
+        prod.post_status('staging.a', 'success')
+    env.run_crons()
+    pr_a_id, pr_b_id, pr_c_id = pr_ids = env['runbot_merge.pull_requests'].search([], order='number')
+    assert len(pr_ids) == 3, \
+        "should have created two forward ports, one in b and one in c (/ master)"
+    # endregion
+
     with prod:
         prod.make_commits(
             'c',
@@ -771,6 +792,15 @@ def test_freeze(env, config, make_repo, users):
         release = prod.make_pr(target='c', head='release-1.1')
     env.run_crons()
 
+    # approve pr_c_id but don't actually merge it before freezing
+    with prod:
+        prod.post_status(pr_b_id.head, 'success')
+        prod.post_status(pr_c_id.head, 'success')
+        prod.get_pr(pr_c_id.number).post_comment('hansen r+', config['role_reviewer']['token'])
+    # review comment should be handled eagerly
+    assert pr_b_id.reviewed_by
+    assert pr_c_id.reviewed_by
+
     w = project.action_prepare_freeze()
     assert w['res_model'] == 'runbot_merge.project.freeze'
     w_id = env[w['res_model']].browse([w['res_id']])
@@ -781,6 +811,8 @@ def test_freeze(env, config, make_repo, users):
     assert not w_id.errors
     w_id.action_freeze()
 
+    assert project.branch_ids.mapped('name') == ['c', 'post-b', 'b', 'a']
+
     # re-enable forward-port cron after freeze
     _, cron_id = env['ir.model.data'].check_object_reference('forwardport', 'port_forward', context={'active_test': False})
     env['ir.cron'].browse([cron_id]).active = True
@@ -792,9 +824,35 @@ def test_freeze(env, config, make_repo, users):
 
     assert release_id.state == 'merged'
     assert not env['runbot_merge.pull_requests'].search([
-        ('state', '!=', 'merged')
+        ('source_id', '=', release_id.id),
     ]), "the release PRs should not be forward-ported"
 
+    assert env['runbot_merge.stagings'].search_count([]) == 2,\
+        "b and c forward ports should be staged since they were ready before freeze"
+
+    # an intermediate PR should have been created
+    pr_inserted = env['runbot_merge.pull_requests'].search([
+        ('source_id', '=', pr_a_id.id),
+        ('target.name', '=', 'post-b'),
+    ])
+    assert pr_inserted, "an intermediate PR should have been reinsered in the sequence"
+    assert pr_c_id.parent_id == pr_inserted
+    assert pr_inserted.parent_id == pr_b_id
+
+    assert pr_inserted.reviewed_by == pr_c_id.reviewed_by,\
+        "review state should have been copied over from c (master)"
+    with prod:
+        prod.post_status(pr_inserted.head, 'success')
+        prod.post_status('staging.b', 'success')
+        prod.post_status('staging.c', 'success')
+    env.run_crons()
+    with prod:
+        prod.post_status('staging.post-b', 'success')
+    env.run_crons()
+
+    assert env['runbot_merge.pull_requests'].search_count([('state', '=', 'merged')]) \
+        == len(['release', 'initial', 'fw-b', 'fw-post-b', 'fw-c'])
+
 def test_missing_magic_ref(env, config, make_repo):
     """There are cases where github fails to create / publish or fails to update
     the magic refs in refs/pull/*.

From f4035932e30fdb308076a510f8086e2f062b0b58 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 7 Jun 2024 15:51:26 +0200
Subject: [PATCH 115/192] [IMP] runbot_merge: add staging status to dashboard

The dashboard can be a bit unclear as to the state of a PR when
everything's gone well. Make it more clear / explicit that it's ready
or staged.

Fixes #888
---
 runbot_merge/controllers/dashboard.py | 5 ++++-
 runbot_merge/views/templates.xml      | 6 ++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/runbot_merge/controllers/dashboard.py b/runbot_merge/controllers/dashboard.py
index 0e19401c..79dd785c 100644
--- a/runbot_merge/controllers/dashboard.py
+++ b/runbot_merge/controllers/dashboard.py
@@ -287,10 +287,12 @@ def raster_render(pr):
                 '' if p['checked'] else 'unchecked',
                 '' if p['reviewed'] else 'unreviewed',
                 '' if p['attached'] else 'detached',
+                'staged' if p['pr'].staging_id else 'ready' if p['pr']._ready else ''
             ]):
                 label = f' {attribute}'
+                color = SUCCESS if attribute in ('staged', 'ready') else ERROR
                 draw.text((offset, top), label,
-                          fill=blend(ERROR, opacity, over=background),
+                          fill=blend(color, opacity, over=background),
                           font=supfont)
                 offset += supfont.getbbox(label)[2]
             offset += math.ceil(supfont.getlength(" "))
@@ -302,6 +304,7 @@ def raster_render(pr):
 Color = Tuple[int, int, int]
 TEXT: Color = (102, 102, 102)
 ERROR: Color = (220, 53, 69)
+SUCCESS: Color = (40, 167, 69)
 BG: Mapping[str | None, Color] = collections.defaultdict(lambda: (255, 255, 255), {
     'info': (217, 237, 247),
     'success': (223, 240, 216),
diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index 6a9c6060..0bb7fcd0 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -597,6 +597,12 @@ action = batches
                                 <sup t-if="not p['attached']"
                                      t-attf-title="detached: {{p['pr'].detach_reason}}"
                                      class="text-warning fa fa-unlink"/>
+                                <sup t-if="p['pr'].staging_id" class="text-success">
+                                    staged
+                                </sup>
+                                <sup t-elif="p['pr']._ready" class="text-success">
+                                    ready
+                                </sup>
                             </span>
                         </td>
                         <td t-else=""/>

From 2fb85c515e4b17fc3e12584a8d70c850cac4ba55 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 7 Jun 2024 17:05:58 +0200
Subject: [PATCH 116/192] [ADD] runbot_merge: missing staging migration

44084e303ccece3cb54128ab29eab399bd4d24e9 changed the interpretation
and schema of the `statuses_cache` field on stagings, but I forgot to
add a migration, so it would just blow up on opening the home
dashboard or the staging lists.
---
 runbot_merge/__manifest__.py                  |  2 +-
 .../migrations/15.0.1.15/pre-migration.py     | 22 +++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 runbot_merge/migrations/15.0.1.15/pre-migration.py

diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py
index 5f23262c..a21ce712 100644
--- a/runbot_merge/__manifest__.py
+++ b/runbot_merge/__manifest__.py
@@ -1,6 +1,6 @@
 {
     'name': 'merge bot',
-    'version': '1.14',
+    'version': '1.15',
     'depends': ['contacts', 'mail', 'website'],
     'data': [
         'security/security.xml',
diff --git a/runbot_merge/migrations/15.0.1.15/pre-migration.py b/runbot_merge/migrations/15.0.1.15/pre-migration.py
new file mode 100644
index 00000000..531811dd
--- /dev/null
+++ b/runbot_merge/migrations/15.0.1.15/pre-migration.py
@@ -0,0 +1,22 @@
+"""Completely missed that in 44084e303ccece3cb54128ab29eab399bd4d24e9 I
+completely changed the semantics and structure of the statuses_cache, so the
+old caches don't actually work anymore at all.
+
+This rewrites all existing caches.
+"""
+def migrate(cr, version):
+    cr.execute("""
+WITH statuses AS (
+    SELECT
+        s.id as staging_id,
+        json_object_agg(c.sha, c.statuses::json) as statuses
+    FROM runbot_merge_stagings s
+    LEFT JOIN runbot_merge_stagings_heads h ON (h.staging_id = s.id)
+    LEFT JOIN runbot_merge_commit c ON (h.commit_id = c.id)
+    GROUP BY s.id
+)
+UPDATE runbot_merge_stagings
+SET statuses_cache = statuses
+FROM statuses
+WHERE id = staging_id
+    """)

From 4af515b20d518031dd6b2af5a8a4e959fd726684 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 7 Jun 2024 17:07:10 +0200
Subject: [PATCH 117/192] [IMP] runbot_merge: stagings button wrapping

Because one of the previous commits adds the duration of the staging
to the staging dropdown toggles, it's now much longer, and by default
the text does not wrap so it looks like shit and goes completely out
the column "CSS is awesome" style.

Update the style of the dropdown toggles specifically to allow text
wrapping. Also align them left instead of centering, because the text
makes a centered layout super ugly.
---
 runbot_merge/static/scss/runbot_merge.scss | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/runbot_merge/static/scss/runbot_merge.scss b/runbot_merge/static/scss/runbot_merge.scss
index e5a18687..92d807a4 100644
--- a/runbot_merge/static/scss/runbot_merge.scss
+++ b/runbot_merge/static/scss/runbot_merge.scss
@@ -82,6 +82,11 @@ $mergebot-colors: ("success": #dff0d8, "danger": #f2dede, "warning": #fcf8e3, "i
   .batch a:not(:last-of-type) a:after {
     content: ",";
   }
+
+  button.dropdown-toggle {
+    text-align: left;
+    white-space: wrap;
+  }
 }
 .pr-listing > * { display: inline-block; }
 .pr-awaiting { opacity: 0.8; }

From 14a2b0068dc47a83cac8623f47c52fc0b4d43e0d Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 10 Jun 2024 14:29:55 +0200
Subject: [PATCH 118/192] [FIX] runbot_merge: type error in conflict handling

---
 runbot_merge/models/pull_requests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 2055a2cf..7cdc6a38 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1430,7 +1430,7 @@ class PullRequests(models.Model):
             template = 'runbot_merge.forwardport.linked'
             format_args = {
                 'pr': self,
-                'siblings': ', '.join(p.display_name for p in (self.batch_id - self)),
+                'siblings': ', '.join(p.display_name for p in (self.batch_id.prs - self)),
                 'footer': FOOTER,
             }
         elif not self._find_next_target():

From e403593799740d3eec278e6eeae72c11dc818020 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 10 Jun 2024 14:31:02 +0200
Subject: [PATCH 119/192] [FIX] runbot_merge: incorrect computation
 dependencies

`Batch.staging_ids` is a computed field, it can't be used as a
dependency for an other compute (at least not in 15.0).
---
 runbot_merge/models/pull_requests.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 7cdc6a38..a40db2d8 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -396,12 +396,12 @@ class PullRequests(models.Model):
     staging_id = fields.Many2one('runbot_merge.stagings', compute='_compute_staging', store=True)
     staging_ids = fields.Many2many('runbot_merge.stagings', string="Stagings", compute='_compute_stagings', context={"active_test": False})
 
-    @api.depends('batch_id.staging_ids.active')
+    @api.depends('batch_id.batch_staging_ids.runbot_merge_stagings_id.active')
     def _compute_staging(self):
         for p in self:
             p.staging_id = p.batch_id.staging_ids.filtered('active')
 
-    @api.depends('batch_id.staging_ids')
+    @api.depends('batch_id.batch_staging_ids.runbot_merge_stagings_id')
     def _compute_stagings(self):
         for p in self:
             p.staging_ids = p.batch_id.staging_ids

From 187f7f642948811c2f6e9d5b0c884de44e56e758 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 10 Jun 2024 15:17:11 +0200
Subject: [PATCH 120/192] [CHG] runbot_merge: allow pr author to approve all fw

- trigger FW section on all forward ports, not just attached ones
- allow author of original PR to approve any fwport
---
 runbot_merge/models/pull_requests.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index a40db2d8..e1a957c6 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -665,7 +665,7 @@ class PullRequests(models.Model):
             match command:
                 case commands.Approve() if self.draft:
                     msg = "draft PRs can not be approved."
-                case commands.Approve() if self.parent_id:
+                case commands.Approve() if self.source_id:
                     # rules are a touch different for forwardport PRs:
                     valid = lambda _: True if command.ids is None else lambda n: n in command.ids
                     _, source_reviewer, source_author = self.source_id._pr_acl(author)
@@ -674,6 +674,10 @@ class PullRequests(models.Model):
                     # - reviewers on the original can approve any forward port
                     if source_reviewer:
                         approveable = ancestors
+                    elif source_author:
+                        # give full review rights on all forwardports (attached
+                        # or not) to original author
+                        approveable = ancestors
                     else:
                         # between the first merged ancestor and self
                         mergeors = list(itertools.dropwhile(

From c67325fdabb4ec4b090df43be0ab96108ba40180 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 10 Jun 2024 18:47:49 +0200
Subject: [PATCH 121/192] [IMP] forwardport: don't ping *every* forwardport

98aaa910 updated the forwardport notifications system to notify on the
forward ports rather than the source, to try and mitigate or at least
shift some of the spam: spam the followers of the original
source (which might be many people) somewhat less, at the possible
cost of spamming the author and reviewer more because they get a
message per forgotten forward port.

This change aims to alleviate part of the latter, by only notifying on
PRs which actually need to be r+'d, and not notifying on those which
will implicitly "inherit" the reviews. This should cut down on
redundant notifications and let users focus on the important ones.
---
 forwardport/models/project.py    |  17 +++--
 forwardport/tests/test_simple.py |   8 +--
 forwardport/tests/test_weird.py  | 103 ++++++++++++++++++++++++++++++-
 3 files changed, 116 insertions(+), 12 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 1ce6abb7..892c2d58 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -11,13 +11,14 @@ means PR creation is trickier (as mergebot assumes opened event will always
 lead to PR creation but fpbot wants to attach meaning to the PR when setting
 it up), ...
 """
+from __future__ import annotations
+
 import datetime
 import itertools
 import json
 import logging
 import operator
 import subprocess
-import sys
 import tempfile
 import typing
 from pathlib import Path
@@ -181,6 +182,8 @@ class PullRequests(models.Model):
     reviewed_by: Partner
     head: str
     state: str
+    merge_date: datetime.datetime
+    parent_id: PullRequests
 
     reminder_backoff_factor = fields.Integer(default=-4, group_operator=None)
 
@@ -529,11 +532,11 @@ stderr:
         # don't stringify so caller can still perform alterations
         return msg
 
-    def _outstanding(self, cutoff):
+    def _outstanding(self, cutoff: str) -> typing.ItemsView[PullRequests, list[PullRequests]]:
         """ Returns "outstanding" (unmerged and unclosed) forward-ports whose
         source was merged before ``cutoff`` (all of them if not provided).
 
-        :param str cutoff: a datetime (ISO-8601 formatted)
+        :param cutoff: a datetime (ISO-8601 formatted)
         :returns: an iterator of (source, forward_ports)
         """
         return groupby(self.env['runbot_merge.pull_requests'].search([
@@ -551,11 +554,17 @@ stderr:
 
         for source, prs in self._outstanding(cutoff):
             backoff = dateutil.relativedelta.relativedelta(days=2**source.reminder_backoff_factor)
-            prs = list(prs)
             if source.merge_date > (cutoff_dt - backoff):
                 continue
             source.reminder_backoff_factor += 1
+
+            # only keep the PRs which don't have an attached descendant)
+            pr_ids = {p.id for p in prs}
             for pr in prs:
+                pr_ids.discard(pr.parent_id.id)
+            print(source, prs, [p.parent_id for p in prs],
+                  '\n\t->', pr_ids, flush=True)
+            for pr in (p for p in prs if p.id in pr_ids):
                 self.env.ref('runbot_merge.forwardport.reminder')._send(
                     repository=pr.repository,
                     pull_request=pr.number,
diff --git a/forwardport/tests/test_simple.py b/forwardport/tests/test_simple.py
index eae53ce4..b17f83d8 100644
--- a/forwardport/tests/test_simple.py
+++ b/forwardport/tests/test_simple.py
@@ -139,13 +139,7 @@ def test_straightforward_flow(env, config, make_repo, users):
 This PR targets b and is part of the forward-port chain. Further PRs will be created up to c.
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
-"""),
-        (users['user'], "@%s @%s this forward port of %s is awaiting action (not merged or closed)." % (
-            users['other'],
-            users['reviewer'],
-            pr0.display_name,
-        ))
-    ]
+""")]
 
     assert pr0_ == pr0
     assert pr1_ == pr1
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index e0922ceb..8bc62aad 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import pytest
 
@@ -1164,3 +1164,104 @@ def test_maintain_batch_history(env, config, make_repo, users):
     assert pr1_b_id in b_batch.all_prs, "the PR is still in the batch"
     assert pr1_b_id not in b_batch.prs, "the PR is not in the open/active batch PRs"
     # endregion
+
+FMT = '%Y-%m-%d %H:%M:%S'
+FAKE_PREV_WEEK = (datetime.now() + timedelta(days=1)).strftime(FMT)
+def test_reminder_detached(env, config, make_repo, users):
+    """On detached forward ports, both sides of the detachment should be notified.
+    """
+    # region setup
+    prod, _ = make_basic(env, config, make_repo, statuses='default')
+    with prod:
+        prod.make_commits('a', Commit('c', tree={'x': '0'}), ref="heads/abranch")
+        pr_a = prod.make_pr(target='a', head='abranch')
+        prod.post_status('abranch', 'success')
+        pr_a.post_comment('hansen r+ fw=skipci', config['role_reviewer']['token'])
+    env.run_crons()
+
+    with prod:
+        prod.post_status('staging.a', 'success')
+    env.run_crons()
+
+    pr_a_id = to_pr(env, pr_a)
+    pr_b_id = env['runbot_merge.pull_requests'].search([
+        ('target.name', '=', 'b'),
+        ('parent_id', '=', pr_a_id.id),
+    ])
+    assert pr_b_id
+    with prod:
+        prod.post_status(pr_b_id.head, 'success')
+    env.run_crons()
+    pr_c_id = env['runbot_merge.pull_requests'].search([
+        ('target.name', '=', 'c'),
+        ('parent_id', '=', pr_b_id.id),
+    ])
+    assert pr_c_id
+    # endregion
+
+    pr_b = prod.get_pr(pr_b_id.number)
+    pr_c = prod.get_pr(pr_c_id.number)
+
+    # region sanity check
+    env.run_crons('forwardport.reminder', 'runbot_merge.feedback_cron', context={'forwardport_updated_before': FAKE_PREV_WEEK})
+
+    assert pr_b.comments == [
+        seen(env, pr_b, users),
+        (users['user'], """\
+This PR targets b and is part of the forward-port chain. Further PRs will be created up to c.
+
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+""")], "the intermediate PR should not be reminded"
+
+    assert pr_c.comments == [
+        seen(env, pr_c, users),
+        (users['user'], """\
+@%s @%s this PR targets c and is the last of the forward-port chain containing:
+* %s
+
+To merge the full chain, use
+> @hansen r+
+
+More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
+""" % (
+            users['user'], users['reviewer'],
+            pr_b_id.display_name,
+    )),
+        (users['user'], "@%s @%s this forward port of %s is awaiting action (not merged or closed)." % (
+            users['user'],
+            users['reviewer'],
+            pr_a_id.display_name,
+        ))
+    ], "the final PR should be reminded"
+    # endregion
+
+    # region check detached
+    pr_c_id.write({'parent_id': False, 'detach_reason': 'because'})
+    env.run_crons('forwardport.reminder', 'runbot_merge.feedback_cron', context={'forwardport_updated_before': FAKE_PREV_WEEK})
+
+    for p in env['runbot_merge.pull_requests'].search_read(
+        [], ['id', 'parent_id', 'source_id', 'display_name']
+    ):
+        print(p, flush=True)
+
+    assert pr_b.comments[2:] == [
+        (users['user'], "@%s @%s child PR %s was modified / updated and has become a normal PR. This PR (and any of its parents) will need to be merged independently as approvals won't cross." % (
+            users['user'],
+            users['reviewer'],
+            pr_c_id.display_name,
+        )),
+        (users['user'], "@%s @%s this forward port of %s is awaiting action (not merged or closed)." % (
+            users['user'],
+            users['reviewer'],
+            pr_a_id.display_name,
+        ))
+    ], "the detached-from intermediate PR should now be reminded"
+    assert pr_c.comments[3:] == [
+        (users['user'], "@%(user)s @%(reviewer)s this PR was modified / updated and has become a normal PR. It must be merged directly." % users),
+        (users['user'], "@%s @%s this forward port of %s is awaiting action (not merged or closed)." % (
+            users['user'],
+            users['reviewer'],
+            pr_a_id.display_name,
+        ))
+    ], "the final forward port should be reminded as before"
+    # endregion

From e320de04397726dbc14492caaf5a0964cd4a23a3 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 11 Jun 2024 15:24:09 +0200
Subject: [PATCH 122/192] [FIX] runbot_merge: handle gh comments ending with
 newlines

Regex `$` apparently does not quite strip that out.
---
 runbot_merge/models/pull_requests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index e1a957c6..4526faa5 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -632,7 +632,7 @@ class PullRequests(models.Model):
             cmds: List[commands.Command] = [
                 ps
                 for line in commandlines
-                for ps in commands.Parser(line)
+                for ps in commands.Parser(line.rstrip())
             ]
         except Exception as e:
             _logger.info(

From 60c4b5141d55ebb73b5f8e4b1f1738c80369722e Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 11 Jun 2024 15:31:35 +0200
Subject: [PATCH 123/192] [FIX] runbot_merge: leftover direct setting of PR
 state

Setting the PR state directly really doesn't work as it doesn't
correctly save (and can get overwritten by any dependency of which
there are many).

This caused setting odoo/odoo#165777 in error to fail, leading to it
being re-staged (and failing) repeatedly, and the PR being spammed
with comments.

- create a more formal helper for preventing directly setting computed
  functions (without an actual inverse)
- replace direct state setting by setting the corresponding dependency
  e.g. `error` for error and `skipchecks` to force a PR to ready
- add a `skipchecks` inverse to the PR so it can also set itself as
  reviewed, and is convenient, might be worth also adding stuff to
  `Batch.write`
---
 runbot_merge/models/pull_requests.py   | 29 ++++++++++++++++++--------
 runbot_merge/models/stagings_create.py |  2 +-
 runbot_merge/models/utils.py           |  4 ++++
 runbot_merge/tests/test_basic.py       |  2 +-
 runbot_merge/tests/test_oddities.py    | 23 +++++++++++++++++---
 5 files changed, 46 insertions(+), 14 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 4526faa5..ae48c14f 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -22,7 +22,7 @@ from odoo.exceptions import AccessError
 from odoo.osv import expression
 from odoo.tools import html_escape, Reverse
 from . import commands
-from .utils import enum
+from .utils import enum, readonly
 
 from .. import github, exceptions, controllers, utils
 
@@ -328,11 +328,11 @@ class PullRequests(models.Model):
 
     closed = fields.Boolean(default=False, tracking=True)
     error = fields.Boolean(string="in error", default=False, tracking=True)
-    skipchecks = fields.Boolean(related='batch_id.skipchecks')
+    skipchecks = fields.Boolean(related='batch_id.skipchecks', inverse='_inverse_skipchecks')
     cancel_staging = fields.Boolean(related='batch_id.cancel_staging')
     merge_date = fields.Datetime(
         related='batch_id.merge_date',
-        inverse=lambda _: 1/0,
+        inverse=readonly,
         tracking=True,
         store=True,
     )
@@ -347,8 +347,13 @@ class PullRequests(models.Model):
         ('merged', 'Merged'),
         ('error', 'Error'),
     ],
-        compute='_compute_state', store=True, default='opened',
-        index=True, tracking=True, column_type=enum(_name, 'state'),
+        compute='_compute_state',
+        inverse=readonly,
+        readonly=True,
+        store=True,
+        index=True,
+        tracking=True,
+        column_type=enum(_name, 'state'),
     )
 
     number = fields.Integer(required=True, index=True, group_operator=None)
@@ -376,7 +381,7 @@ class PullRequests(models.Model):
 
     reviewed_by = fields.Many2one('res.partner', index=True, tracking=True)
     delegates = fields.Many2many('res.partner', help="Delegate reviewers, not intrinsically reviewers but can review this PR")
-    priority = fields.Selection(related="batch_id.priority", inverse=lambda _: 1 / 0)
+    priority = fields.Selection(related="batch_id.priority", inverse=readonly)
 
     overrides = fields.Char(required=True, default='{}', tracking=True)
     statuses = fields.Text(help="Copy of the statuses from the HEAD commit, as a Python literal", default="{}")
@@ -389,12 +394,12 @@ class PullRequests(models.Model):
         ('pending', 'Pending'),
         ('failure', 'Failure'),
         ('success', 'Success'),
-    ], compute='_compute_statuses', store=True, column_type=enum(_name, 'status'))
+    ], compute='_compute_statuses', store=True, inverse=readonly, column_type=enum(_name, 'status'))
     previous_failure = fields.Char(default='{}')
 
     batch_id = fields.Many2one('runbot_merge.batch', index=True)
-    staging_id = fields.Many2one('runbot_merge.stagings', compute='_compute_staging', store=True)
-    staging_ids = fields.Many2many('runbot_merge.stagings', string="Stagings", compute='_compute_stagings', context={"active_test": False})
+    staging_id = fields.Many2one('runbot_merge.stagings', compute='_compute_staging', inverse=readonly, store=True)
+    staging_ids = fields.Many2many('runbot_merge.stagings', string="Stagings", compute='_compute_stagings', inverse=readonly, context={"active_test": False})
 
     @api.depends('batch_id.batch_staging_ids.runbot_merge_stagings_id.active')
     def _compute_staging(self):
@@ -486,6 +491,12 @@ class PullRequests(models.Model):
     def _compute_display_name(self):
         return super()._compute_display_name()
 
+    def _inverse_skipchecks(self):
+        for p in self:
+            p.batch_id.skipchecks = p.skipchecks
+            if p.skipchecks:
+                p.reviewed_by = self.env.user.partner_id
+
     def name_get(self):
         name_template = '%(repo_name)s#%(number)d'
         if self.env.context.get('pr_include_title'):
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index ec3d99b3..ddc9c3d4 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -270,7 +270,7 @@ def stage_batches(branch: Branch, batches: Batch, staging_state: StagingState) -
                 with contextlib.suppress(Exception):
                     reason = json.loads(str(reason))['message'].lower()
 
-                pr.state = 'error'
+                pr.error = True
                 env.ref('runbot_merge.pr.merge.failed')._send(
                     repository=pr.repository,
                     pull_request=pr.number,
diff --git a/runbot_merge/models/utils.py b/runbot_merge/models/utils.py
index 6aeeb9b8..f91eb080 100644
--- a/runbot_merge/models/utils.py
+++ b/runbot_merge/models/utils.py
@@ -4,3 +4,7 @@ from typing import Tuple
 def enum(model: str, field: str) -> Tuple[str, str]:
     n = f'{model.replace(".", "_")}_{field}_type'
     return n, n
+
+
+def readonly(_):
+    raise TypeError("Field is readonly")
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 125d1477..53c069c9 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -2988,7 +2988,7 @@ class TestBatching(object):
             pr01 = self._pr(repo, 'Urgent1', [{'n': 'n'}, {'o': 'o'}], user=config['role_user']['token'], reviewer=None, statuses=[])
             pr01.post_comment('hansen NOW!', config['role_reviewer']['token'])
         p_01 = to_pr(env, pr01)
-        p_01.state = 'error'
+        p_01.error = True
 
         env.run_crons()
         assert not p_01.staging_id, "p_01 should not be picked up as it's failed"
diff --git a/runbot_merge/tests/test_oddities.py b/runbot_merge/tests/test_oddities.py
index 8b2bba21..0971b8f7 100644
--- a/runbot_merge/tests/test_oddities.py
+++ b/runbot_merge/tests/test_oddities.py
@@ -211,16 +211,22 @@ def test_merge_emptying_commits(env, repo, users, config):
 
     ping = f"@{users['user']} @{users['reviewer']}"
     # check that first / sole commit emptying is caught
-    assert not to_pr(env, pr1).staging_id
+    pr1_id = to_pr(env, pr1)
+    assert not pr1_id.staging_id
     assert pr1.comments[3:] == [
         (users['user'], f"{ping} unable to stage: commit {c1} results in an empty tree when merged, it is likely a duplicate of a merged commit, rebase and remove.")
     ]
+    assert pr1_id.error
+    assert pr1_id.state == 'error'
 
     # check that followup commit emptying is caught
-    assert not to_pr(env, pr2).staging_id
+    pr2_id = to_pr(env, pr2)
+    assert not pr2_id.staging_id
     assert pr2.comments[3:] == [
         (users['user'], f"{ping} unable to stage: commit {c2} results in an empty tree when merged, it is likely a duplicate of a merged commit, rebase and remove.")
     ]
+    assert pr2_id.error
+    assert pr2_id.state == 'error'
 
     # check that emptied squashed pr is caught
     pr3_id = to_pr(env, pr3)
@@ -228,6 +234,17 @@ def test_merge_emptying_commits(env, repo, users, config):
     assert pr3.comments[3:] == [
         (users['user'], f"{ping} unable to stage: results in an empty tree when merged, might be the duplicate of a merged PR.")
     ]
+    assert pr3_id.error
+    assert pr3_id.state == 'error'
+
+    # ensure the PR does not get re-staged since it's the first of the staging
+    # (it's the only one)
+    env.run_crons()
+    assert pr1.comments[3:] == [
+        (users['user'], f"{ping} unable to stage: commit {c1} results in an empty tree when merged, it is likely a duplicate of a merged commit, rebase and remove.")
+    ]
+    assert len(pr2.comments) == 4
+    assert len(pr3.comments) == 4
 
 def test_force_ready(env, repo, config):
     with repo:
@@ -238,7 +255,7 @@ def test_force_ready(env, repo, config):
     env.run_crons()
 
     pr_id = to_pr(env, pr)
-    pr_id.state = 'ready'
+    pr_id.skipchecks = True
 
     assert pr_id.state == 'ready'
     assert pr_id.status == 'pending'

From 2ab06ca96b6b324823305c2cdf4f4447592926bf Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 11 Jun 2024 15:41:03 +0200
Subject: [PATCH 124/192] [IMP] *: require explicitly specifying whether
 exceptions in logs are valid

Seems like a good idea to better keep track of the log of an Odoo used
to testing, and avoid silently ignoring logged errors.

- intercept odoo's stderr via a pipe, that way we can still write it
  back out and pytest is able to read & buffer it, pytest's capfd
  would not work correctly: it breaks output capturing (and printing
  on failure); and because of the way it hooks in it's unable to
  capture from subprocesses inheriting the standard stream, cf
  pytest-dev/pytest#4428
- update the env fixture to check that the odoo log doesn't have any
  exception on failure
- make that check conditional on the `expect_log_errors` marker, this
  way we can mark tests for which we expect errors to be logged, and
  assert that that does happen
---
 conftest.py                          | 36 ++++++++++++++++++++++++----
 forwardport/tests/test_conflicts.py  |  4 ++++
 forwardport/tests/test_limit.py      |  1 +
 forwardport/tests/test_weird.py      |  2 ++
 runbot_merge/tests/test_basic.py     |  9 +++++++
 runbot_merge/tests/test_multirepo.py |  2 ++
 runbot_merge/tests/test_oddities.py  |  3 +++
 7 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/conftest.py b/conftest.py
index 946c9466..21af5274 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import select
+import threading
 from typing import Optional
 
 """
@@ -94,9 +96,12 @@ def pytest_addoption(parser):
 def is_manager(config):
     return not hasattr(config, 'workerinput')
 
-# noinspection PyUnusedLocal
 def pytest_configure(config):
     sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'mergebot_test_utils'))
+    config.addinivalue_line(
+        "markers",
+        "expect_log_errors(reason): allow and require tracebacks in the log",
+    )
 
 def pytest_unconfigure(config):
     if not is_manager(config):
@@ -418,6 +423,19 @@ def server(request, db, port, module, addons_path, tmpdir):
     if request.config.getoption('--coverage'):
         cov = ['coverage', 'run', '-p', '--source=odoo.addons.runbot_merge,odoo.addons.forwardport', '--branch']
 
+    r, w = os.pipe2(os.O_NONBLOCK)
+    buf = bytearray()
+    def _move(inpt=r, output=sys.stdout.fileno()):
+        while p.poll() is None:
+            readable, _, _ = select.select([inpt], [], [], 1)
+            if readable:
+                r = os.read(inpt, 4096)
+                if not r:
+                    break
+                os.write(output, r)
+                buf.extend(r)
+        os.close(inpt)
+
     p = subprocess.Popen([
         *cov,
         'odoo', '--http-port', str(port),
@@ -425,25 +443,35 @@ def server(request, db, port, module, addons_path, tmpdir):
         '-d', db,
         '--max-cron-threads', '0', # disable cron threads (we're running crons by hand)
         *itertools.chain.from_iterable(('--log-handler', h) for h in log_handlers),
-    ], env={
+    ], stderr=w, env={
         **os.environ,
         # stop putting garbage in the user dirs, and potentially creating conflicts
         # TODO: way to override this with macOS?
         'XDG_DATA_HOME': str(tmpdir.mkdir('share')),
         'XDG_CACHE_HOME': str(tmpdir.mkdir('cache')),
     })
+    os.close(w)
+    # start the reader thread here so `_move` can read `p` without needing
+    # additional handholding
+    threading.Thread(target=_move, daemon=True).start()
 
     try:
         wait_for_server(db, port, p, module)
 
-        yield p
+        yield p, buf
     finally:
         p.terminate()
         p.wait(timeout=30)
 
 @pytest.fixture
-def env(port, server, db, default_crons):
+def env(request, port, server, db, default_crons):
     yield Environment(port, db, default_crons)
+    if request.node.get_closest_marker('expect_log_errors'):
+        if b"Traceback (most recent call last):" not in server[1]:
+            pytest.fail("should have found error in logs.")
+    else:
+        if b"Traceback (most recent call last):" in server[1]:
+            pytest.fail("unexpected error in logs, fix, or mark function as `expect_log_errors` to require.")
 
 def check(response):
     assert response.ok, response.text or response.reason
diff --git a/forwardport/tests/test_conflicts.py b/forwardport/tests/test_conflicts.py
index eed2bcc0..4172f988 100644
--- a/forwardport/tests/test_conflicts.py
+++ b/forwardport/tests/test_conflicts.py
@@ -2,6 +2,8 @@ import re
 import time
 from operator import itemgetter
 
+import pytest
+
 from utils import make_basic, Commit, validate_all, matches, seen, REF_PATTERN, to_pr
 
 
@@ -296,6 +298,8 @@ def test_multiple_commits_same_authorship(env, config, make_repo):
     assert get(c.author) == get(author)
     assert get(c.committer) == get(committer)
 
+
+@pytest.mark.expect_log_errors(reason="commits with incomplete authorship are unmergeable, which is logged as error")
 def test_multiple_commits_different_authorship(env, config, make_repo, users, rolemap):
     """ When a PR has multiple commits by different authors, the resulting
     (squashed) conflict commit should have an empty email
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index 673a2bcf..9d88e42a 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -56,6 +56,7 @@ def test_ignore(env, config, make_repo):
         "should not have created a forward port"
 
 
+@pytest.mark.expect_log_errors(reason="one of the limit-setting does not provide a branch name")
 def test_disable(env, config, make_repo, users):
     """ Checks behaviour if the limit target is disabled:
 
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index 8bc62aad..d6cd732d 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -853,6 +853,8 @@ def test_freeze(env, config, make_repo, users):
     assert env['runbot_merge.pull_requests'].search_count([('state', '=', 'merged')]) \
         == len(['release', 'initial', 'fw-b', 'fw-post-b', 'fw-c'])
 
+
+@pytest.mark.expect_log_errors(reason="missing / invalid head causes an error to be logged")
 def test_missing_magic_ref(env, config, make_repo):
     """There are cases where github fails to create / publish or fails to update
     the magic refs in refs/pull/*.
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 53c069c9..6dec9506 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -482,6 +482,8 @@ def test_staging_concurrent(env, repo, config):
     ])
     assert pr2.staging_id
 
+
+@pytest.mark.expect_log_errors(reason="staging merge conflicts are logged")
 def test_staging_conflict_first(env, repo, users, config, page):
     """ If the first batch of a staging triggers a conflict, the PR should be
     marked as in error
@@ -512,6 +514,8 @@ def test_staging_conflict_first(env, repo, users, config, page):
     assert dangerbox
     assert dangerbox[0].text.strip() == 'Unable to stage PR'
 
+
+@pytest.mark.expect_log_errors(reason="merge conflicts are logged as errors")
 def test_staging_conflict_second(env, repo, users, config):
     """ If the non-first batch of a staging triggers a conflict, the PR should
     just be skipped: it might be a conflict with an other PR which could fail
@@ -634,6 +638,8 @@ def test_staging_ci_failure_single(env, repo, users, config, page):
     assert dangerbox
     assert dangerbox[0].text == 'ci/runbot'
 
+
+@pytest.mark.expect_log_errors(reason="failed fast forward of staging is a logged error")
 def test_ff_failure(env, repo, config, page):
     """ target updated while the PR is being staged => redo staging """
     with repo:
@@ -679,6 +685,8 @@ def test_ff_failure(env, repo, config, page):
     assert repo.commit('heads/staging.master').id != staging.id,\
         "PR should be staged to a new commit"
 
+
+@pytest.mark.expect_log_errors(reason="blocking fast-forward of staging which triggers a logged error when trying to patch the GH ref")
 def test_ff_failure_batch(env, repo, users, config):
     with repo:
         m = repo.make_commit(None, 'initial', None, tree={'m': 'm'})
@@ -3619,6 +3627,7 @@ class TestRecognizeCommands:
             prx.post_comment('%shansen r+' % indent, config['role_reviewer']['token'])
         assert pr.state == 'approved'
 
+    @pytest.mark.expect_log_errors(reason="unknown commands are logged")
     def test_unknown_commands(self, repo, env, config, users):
         with repo:
             m = repo.make_commit(None, 'initial', None, tree={'m': 'm'})
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index e6073028..07e42786 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -396,6 +396,8 @@ def test_sub_match(env, project, repo_a, repo_b, repo_c, config):
     )
     assert s == list(st.ids)
 
+
+@pytest.mark.expect_log_errors(reason="merge errors are logged as errors")
 def test_merge_fail(env, project, repo_a, repo_b, users, config):
     """ In a matched-branch scenario, if merging in one of the linked repos
     fails it should revert the corresponding merges
diff --git a/runbot_merge/tests/test_oddities.py b/runbot_merge/tests/test_oddities.py
index 0971b8f7..081bc3e3 100644
--- a/runbot_merge/tests/test_oddities.py
+++ b/runbot_merge/tests/test_oddities.py
@@ -1,3 +1,4 @@
+import pytest
 import requests
 
 from utils import Commit, to_pr
@@ -171,6 +172,8 @@ def test_merge_empty_commits(env, repo, users, config):
     assert commits[1]['commit']['message'].startswith('thing1')
     assert commits[2]['commit']['message'] == 'initial'
 
+
+@pytest.mark.expect_log_errors(reason="emptied commits are merge errors and logged")
 def test_merge_emptying_commits(env, repo, users, config):
     """The mergebot should *not* allow merging non-empty commits which become
     empty as part of the staging (rebasing)

From d2e730c39b770fdfdea4880d0769a20724f4b29b Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 12 Jun 2024 11:14:44 +0200
Subject: [PATCH 125/192] [IMP] runbot_merge: log ACL error in PR controller

Currently this just silently returns a 404. Since repos are gated by
default (only accessible to internal users) this can get very
confusing when trying to setup a new repo or when forgetting this
information when writing tests.
---
 runbot_merge/controllers/dashboard.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/runbot_merge/controllers/dashboard.py b/runbot_merge/controllers/dashboard.py
index 79dd785c..7b62e9d7 100644
--- a/runbot_merge/controllers/dashboard.py
+++ b/runbot_merge/controllers/dashboard.py
@@ -7,6 +7,7 @@ import colorsys
 import hashlib
 import io
 import json
+import logging
 import math
 import pathlib
 from email.utils import formatdate
@@ -22,6 +23,9 @@ from PIL import Image, ImageDraw, ImageFont
 from odoo.http import Controller, route, request
 from odoo.tools import file_open
 
+
+_logger = logging.getLogger(__name__)
+
 LIMIT = 20
 class MergebotDashboard(Controller):
     @route('/runbot_merge', auth="public", type="http", website=True, sitemap=True)
@@ -101,6 +105,12 @@ class MergebotDashboard(Controller):
         if not pr_id:
             raise werkzeug.exceptions.NotFound()
         if not pr_id.repository.group_id <= request.env.user.groups_id:
+            _logger.warning(
+                "Access error: %s (%s) tried to access %s but lacks access",
+                request.env.user.login,
+                request.env.user.name,
+                pr_id.display_name,
+            )
             raise werkzeug.exceptions.NotFound()
 
         if png:

From 81f133de156547a1f4bbb1f1e1592932d3097023 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 12 Jun 2024 11:16:14 +0200
Subject: [PATCH 126/192] [IMP] test utils: don't ACL-protect repos by default

This is quite frustrating when trying to access the frontend from a
test especially combined with the lack of in-log feedback (before the
previous commit).
---
 mergebot_test_utils/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mergebot_test_utils/utils.py b/mergebot_test_utils/utils.py
index bc451293..c5473e93 100644
--- a/mergebot_test_utils/utils.py
+++ b/mergebot_test_utils/utils.py
@@ -158,6 +158,7 @@ def make_basic(
         'name': prod.name,
         'required_statuses': statuses,
         'fp_remote_target': other.name if other else False,
+        'group_id': False,
     })
     env['res.partner'].search([
         ('github_login', '=', config['role_reviewer']['user'])

From d010f0374aff3e844e02eac08afc41101d2f9b75 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 12 Jun 2024 15:09:47 +0200
Subject: [PATCH 127/192] [FIX] *: dashboard when PRs have different limits

The code selecting the lower and upper bounds for the PR dashboard did
not deal correctly with getting multiple limits in the same genealogy.
---
 forwardport/tests/test_limit.py  | 29 +++++++++++++++++++++--------
 runbot_merge/views/templates.xml |  6 ++++--
 2 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index 9d88e42a..b0f469db 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -1,5 +1,5 @@
-
 import pytest
+import requests
 
 from utils import seen, Commit, make_basic, to_pr
 
@@ -9,28 +9,41 @@ from utils import seen, Commit, make_basic, to_pr
     pytest.param('b', 'b', 0, id='current'),
     pytest.param('b', 'a', 0, id='earlier'),
 ])
-def test_configure_fp_limit(env, config, make_repo, source, limit, count):
-    prod, other = make_basic(env, config, make_repo)
+def test_configure_fp_limit(env, config, make_repo, source, limit, count, port):
+    prod, other = make_basic(env, config, make_repo, statuses="default")
     with prod:
         [c] = prod.make_commits(
             source, Commit('c', tree={'f': 'g'}),
             ref='heads/branch',
         )
         pr = prod.make_pr(target=source, head='branch')
-        prod.post_status(c, 'success', 'legal/cla')
-        prod.post_status(c, 'success', 'ci/runbot')
+        prod.post_status(c, 'success')
         pr.post_comment(f'hansen r+ up to {limit}', config['role_reviewer']['token'])
     env.run_crons()
     with prod:
-        prod.post_status(f'staging.{source}', 'success', 'legal/cla')
-        prod.post_status(f'staging.{source}', 'success', 'ci/runbot')
+        prod.post_status(f'staging.{source}', 'success')
     env.run_crons()
 
+    pr_id = to_pr(env, pr)
     descendants = env['runbot_merge.pull_requests'].search([
-        ('source_id', '=', to_pr(env, pr).id)
+        ('source_id', '=', pr_id.id)
     ])
     assert len(descendants) == count
+    limit_id = env['runbot_merge.branch'].search([('name', '=', limit)])
 
+    assert pr_id.limit_id == limit_id
+    assert not descendants.limit_id, "descendant should not inherit the limit explicitly"
+
+    # check that the basic thingie works
+    r = requests.get(f'http://localhost:{port}/{prod.name}/pull/{pr.number}.png')
+    assert r.ok, r.text
+
+    if descendants:
+        c = env['runbot_merge.branch'].search([('name', '=', 'c')])
+        descendants.limit_id = c.id
+
+        r = requests.get(f'http://localhost:{port}/{prod.name}/pull/{pr.number}.png')
+        assert r.ok
 
 def test_ignore(env, config, make_repo):
     """ Provide an "ignore" command which is equivalent to setting the limit
diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index 0bb7fcd0..ce661eb9 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -462,10 +462,12 @@ elif all(p.state in ('merged', 'closed') for p in genealogy[-1].all_prs):
 else:
     # if the tip of the genealogy is not closed, extend to the furthest limit,
     # keeping branches which are active or have an associated batch / PR
-    limit = genealogy.prs.limit_id.sorted(lambda b: (b.sequence, b.name))
+    limit = min(genealogy.prs.limit_id, key=lambda b: (b.sequence, b.name), default=None)
     limit_high = project.branch_ids.ids.index(limit.id) if limit else None
-    limit = targets.sorted(lambda b: (b.sequence, b.name))[-1]
+
+    limit = max(targets, key=lambda b: (b.sequence, b.name))
     limit_low = project.branch_ids.ids.index(limit.id)
+
     branches = project.branch_ids[limit_high:limit_low+1].filtered(lambda b: b.active or b in targets)[::-1]
 
 action = (project, repos, branches, genealogy)

From a2d7180216845d2bc55bbfae4ac99eaf637abde7 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 12 Jun 2024 15:30:44 +0200
Subject: [PATCH 128/192] [IMP] runbot_merge: move limit to fwport tab

And filter it to only consider branches in the same project as the PR,
and a lower sequence than its target. That way it's harder to fuck up
when trying to set limits from the backend.
---
 runbot_merge/models/pull_requests.py | 2 ++
 runbot_merge/views/mergebot.xml      | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index ae48c14f..53386012 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -323,7 +323,9 @@ class PullRequests(models.Model):
     display_name: str
 
     target = fields.Many2one('runbot_merge.branch', required=True, index=True, tracking=True)
+    target_sequence = fields.Integer(related='target.sequence')
     repository = fields.Many2one('runbot_merge.repository', required=True)
+    project = fields.Many2one(related='repository.project_id')
     # NB: check that target & repo have same project & provide project related?
 
     closed = fields.Boolean(default=False, tracking=True)
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index c1d7418a..3de9738e 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -129,6 +129,8 @@
                     <field name="url" widget="url" class="btn btn-secondary" text="Frontend"/></span>
                 </div>
                 <sheet>
+                    <field name="project" invisible="1"/>
+                    <field name="target_sequence" invisible="1"/>
                     <div class="oe_title">
                         <h1>
                             <field name="repository"/>#<field name="number"/>
@@ -187,7 +189,6 @@
                                     <field name="merge_method"/>
                                     <field name="squash"/>
                                     <field name="draft"/>
-                                    <field name="limit_id"/>
                                 </group>
                                 <group>
                                     <field name="priority"/>
@@ -224,6 +225,7 @@
                         <page name="porting" string="Forward-Porting">
                             <group>
                                 <group>
+                                    <field name="limit_id" domain="[('project_id', '=', project), ('sequence', '&lt;=', target_sequence)]"/>
                                     <field string="Original PR" name="source_id"/>
                                     <field name="parent_id"/>
                                     <field

From 413027ad5bae65552e3857a9c31ed16844c06f14 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 12 Jun 2024 15:45:27 +0200
Subject: [PATCH 129/192] [IMP] runbot_merge: formatting & langage of PR
 attributes

Replace the unclear "unchecked" and "unreviewed" by "missing statuses"
and "missing r+", which are hopefully clearer as they better match
other lingo.

Also increase font for attributes, as size 10 was a bit small.

And finally add staging state to caching key, to differentiate "ready"
from "staged" pictures in gh's cache. "ready" should not be necessary
as it ought be implied by the label.
---
 runbot_merge/controllers/dashboard.py | 15 ++++++++-------
 runbot_merge/views/templates.xml      |  4 ++--
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/runbot_merge/controllers/dashboard.py b/runbot_merge/controllers/dashboard.py
index 7b62e9d7..6a5dd2de 100644
--- a/runbot_merge/controllers/dashboard.py
+++ b/runbot_merge/controllers/dashboard.py
@@ -173,7 +173,7 @@ def raster_render(pr):
     with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Regular.ttf', 'rb') as f:
         font = ImageFont.truetype(f, size=16, layout_engine=0)
         f.seek(0)
-        supfont = ImageFont.truetype(f, size=10, layout_engine=0)
+        supfont = ImageFont.truetype(f, size=13, layout_engine=0)
     with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Bold.ttf', 'rb') as f:
         bold = ImageFont.truetype(f, size=16, layout_engine=0)
 
@@ -203,9 +203,9 @@ def raster_render(pr):
         # technically label (state + blocked) does not actually impact image
         # render (though subcomponents of state do) however blocked is useful
         # to force an etag miss so keeping it
-        # TODO: blocked includes draft & merge method, maybe should change looks?
+
         etag.update(''.join(
-            f"(PS){p['label']},{p['closed']},{p['number']},{p['checked']},{p['reviewed']},{p['attached']}"
+            f"(PS){p['label']},{p['closed']},{p['number']},{p['checked']},{p['reviewed']},{p['attached']},{p['pr'].staging_id.id}"
             for p in ps['prs']
         ).encode())
 
@@ -214,9 +214,10 @@ def raster_render(pr):
             _, _, ww, hh = font.getbbox(f" #{p['number']}")
             w += ww + supfont.getbbox(' '.join(filter(None, [
                 'error' if p['pr'].error else '',
-                '' if p['checked'] else 'unchecked',
-                '' if p['reviewed'] else 'unreviewed',
+                '' if p['checked'] else 'missing statuses',
+                '' if p['reviewed'] else 'missing r+',
                 '' if p['attached'] else 'detached',
+                'staged' if p['pr'].staging_id else 'ready' if p['pr']._ready else '',
             ])))[2]
             h = max(hh, h)
         rows[b] = max(rows.get(b, 0), h)
@@ -294,8 +295,8 @@ def raster_render(pr):
                 draw.line([(left, bottom-h), (left+w, bottom-h)], fill=ERROR)
             for attribute in filter(None, [
                 'error' if p['pr'].error else '',
-                '' if p['checked'] else 'unchecked',
-                '' if p['reviewed'] else 'unreviewed',
+                '' if p['checked'] else 'missing statuses',
+                '' if p['reviewed'] else 'missing r+',
                 '' if p['attached'] else 'detached',
                 'staged' if p['pr'].staging_id else 'ready' if p['pr']._ready else ''
             ]):
diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index ce661eb9..cdad7aa3 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -594,8 +594,8 @@ action = batches
                                    t-attf-class="fa fa-external-link"
                                    t-att-href="p['backend_url']"
                                 />
-                                <sup t-if="not p['checked']" class="text-danger">unchecked</sup>
-                                <sup t-if="not p['reviewed']" class="text-danger">unreviewed</sup>
+                                <sup t-if="not p['checked']" class="text-danger">missing statuses</sup>
+                                <sup t-if="not p['reviewed']" class="text-danger">missing r+</sup>
                                 <sup t-if="not p['attached']"
                                      t-attf-title="detached: {{p['pr'].detach_reason}}"
                                      class="text-warning fa fa-unlink"/>

From 7711d09854c0a28f3cba6a544b97b6b10b289603 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 12 Jun 2024 16:08:25 +0200
Subject: [PATCH 130/192] [IMP] *: add fw=no, deprecate ignore

Without fw-bot being its bearer, "ignore" is a lot less clear than it
used to as it looks to be asking to ignore the PR entirely (as if it
was targeted to an unmanaged branch).

Deprecate this command, and tack on the shortcut to the fw
subcommand. It is slightly sub-par as technically it does not quite
fit with the other subcommands, and furthermore can't be disabled via
fw=default... although maybe it could be? Maybe instead of setting the
limit fw=no could set that value to the forwardport mode, and the
fw_policy users could check that? It would require some more finessing
tho:

- `DEFAULT` would need to be accessible to the author as well as the
  reviewers so the author could toggle between `NO` and `DEFAULT`.
- There should probably be a warning of some sort when setting a limit
  to an unportable PR.
- The dashboards would need to take `NO` in account (though I guess
  that's just defaulting the limit to the target).
---
 forwardport/tests/test_limit.py      | 24 +++++++++++++++++-------
 runbot_merge/models/commands.py      |  1 +
 runbot_merge/models/pull_requests.py | 21 ++++++++++++++++++++-
 3 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index b0f469db..14645c26 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -45,29 +45,39 @@ def test_configure_fp_limit(env, config, make_repo, source, limit, count, port):
         r = requests.get(f'http://localhost:{port}/{prod.name}/pull/{pr.number}.png')
         assert r.ok
 
-def test_ignore(env, config, make_repo):
+def test_ignore(env, config, make_repo, users):
     """ Provide an "ignore" command which is equivalent to setting the limit
     to target
     """
-    prod, other = make_basic(env, config, make_repo)
+    prod, _ = make_basic(env, config, make_repo, statuses="default")
     branch_a = env['runbot_merge.branch'].search([('name', '=', 'a')])
     with prod:
         [c] = prod.make_commits('a', Commit('c', tree={'0': '0'}), ref='heads/mybranch')
         pr = prod.make_pr(target='a', head='mybranch')
-        prod.post_status(c, 'success', 'legal/cla')
-        prod.post_status(c, 'success', 'ci/runbot')
-        pr.post_comment('hansen r+ ignore', config['role_reviewer']['token'])
+        prod.post_status(c, 'success')
+    env.run_crons()
+    with prod:
+        pr.post_comment('hansen ignore', config['role_reviewer']['token'])
+        pr.post_comment('hansen r+ fw=no', config['role_reviewer']['token'])
     env.run_crons()
     pr_id = env['runbot_merge.pull_requests'].search([('number', '=', pr.number)])
     assert pr_id.limit_id == branch_a
 
     with prod:
-        prod.post_status('staging.a', 'success', 'legal/cla')
-        prod.post_status('staging.a', 'success', 'ci/runbot')
+        prod.post_status('staging.a', 'success')
+    env.run_crons()
 
     assert env['runbot_merge.pull_requests'].search([]) == pr_id,\
         "should not have created a forward port"
 
+    assert pr.comments == [
+        seen(env, pr, users),
+        (users['reviewer'], "hansen ignore"),
+        (users['reviewer'], "hansen r+ fw=no"),
+        (users['user'], "'ignore' is deprecated, use 'fw=no' to disable forward porting."),
+        (users['user'], "Forward-port disabled."),
+        (users['user'], "Forward-port disabled."),
+    ]
 
 @pytest.mark.expect_log_errors(reason="one of the limit-setting does not provide a branch name")
 def test_disable(env, config, make_repo, users):
diff --git a/runbot_merge/models/commands.py b/runbot_merge/models/commands.py
index 0f46251f..2911638b 100644
--- a/runbot_merge/models/commands.py
+++ b/runbot_merge/models/commands.py
@@ -137,6 +137,7 @@ class SkipChecks:
 
 class FW(enum.Enum):
     DEFAULT = enum.auto()
+    NO = enum.auto()
     SKIPCI = enum.auto()
     SKIPMERGE = enum.auto()
 
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 53386012..0656c851 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -812,7 +812,24 @@ class PullRequests(models.Model):
                 case commands.Close() if source_author:
                     feedback(close=True)
                 case commands.FW():
-                    if source_reviewer or is_reviewer:
+                    if command == commands.FW.NO:
+                        if is_author:
+                            for p in self.batch_id.prs:
+                                ping, m = p._maybe_update_limit(self.target.name)
+
+                                if ping and p == self:
+                                    msg = m
+                                else:
+                                    if ping:
+                                        m = f"@{login} {m}"
+                                    self.env['runbot_merge.pull_requests.feedback'].create({
+                                        'repository': p.repository.id,
+                                        'pull_request': p.number,
+                                        'message': m,
+                                    })
+                        else:
+                            msg = "you can't set a forward-port limit."
+                    elif source_reviewer or is_reviewer:
                         (self.source_id or self).batch_id.fw_policy = command.name.lower()
                         match command:
                             case commands.FW.DEFAULT:
@@ -825,6 +842,8 @@ class PullRequests(models.Model):
                     else:
                         msg = "you can't configure forward-port CI."
                 case commands.Limit(branch) if is_author:
+                    if branch is None:
+                        feedback(message="'ignore' is deprecated, use 'fw=no' to disable forward porting.")
                     limit = branch or self.target.name
                     for p in self.batch_id.prs:
                         ping, m = p._maybe_update_limit(limit)

From 2662411b96f51d7ba3fcf992f6f307105d89ebb2 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 13 Jun 2024 07:55:42 +0200
Subject: [PATCH 131/192] [FIX] runbot_merge: `_schedule_fp_followup` not
 handling multiple batches

`_schedule_fp_followup` correctly iterates on `self`, however some of
the per-iteration work did not handle that correctly, and would try to
access fields on `self`.

Thankfully in most cases it only works on one batch at a time
anyway, *however* if multiple PRs share a HEAD (which is weird but...)
then `_validate` is called on multiple PRs, which through the
forwardport override leads to `_schedule_fp_followup` being called on
multiple batches, and failing when trying to access the `fw_policy`.

Fix by avoiding the misuse of `self` in the two locations where it's
doing something other than accessing `env`.
---
 runbot_merge/models/batch.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index 00886b1d..fbca57fc 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -427,7 +427,7 @@ class Batch(models.Model):
             if not (batch.parent_id and all(p.parent_id for p in batch.prs)):
                 _logger.info('-> no parent %s (%s)', batch, prs)
                 continue
-            if not self.env.context.get('force_fw') and self.source.fw_policy != 'skipci' \
+            if not self.env.context.get('force_fw') and batch.source.fw_policy != 'skipci' \
                     and (invalid := batch.prs.filtered(lambda p: p.state not in ['validated', 'ready'])):
                 _logger.info(
                     '-> wrong state %s (%s)',
@@ -437,7 +437,7 @@ class Batch(models.Model):
                 continue
 
             # check if we've already forward-ported this branch
-            next_target = self._find_next_targets()
+            next_target = batch._find_next_targets()
             if not next_target:
                 _logger.info("-> forward port done (no next target)")
                 continue

From 9d9cae1d578f7150aa3e6de7ff2f5dd3549cf6e1 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 13 Jun 2024 09:35:29 +0200
Subject: [PATCH 132/192] [FIX] runbot_merge: access to self in loop

This is a low issue as the prs of a commit are only listed from the
form so the compute is pretty much always called with a single record,
but still an unforced error which can easily be fixed.
---
 runbot_merge/models/pull_requests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 0656c851..8a1bd658 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1810,7 +1810,7 @@ class Commit(models.Model):
     def _compute_prs(self):
         for c in self:
             c.pull_requests = self.env['runbot_merge.pull_requests'].search([
-                ('head', '=', self.sha),
+                ('head', '=', c.sha),
             ])
 
 

From 728524db12c8e9a49850081fb0137d39bac8c91a Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 13 Jun 2024 13:30:07 +0200
Subject: [PATCH 133/192] [IMP] runbot_merge: send merge method warning faster,
 and on review

- Instead of warning about the merge method on ready PRs, also warn on
  *approved* (but exclude staged just cuz), as that's really when the
  user wants to know that they forgot to set the merge method
- The cron only triggers hourly, but *if* a user approves a PR *and*
  the merge method is not set yet, chances are good they'll need a
  reminder (if they `r+ rebase-merge` or w/e the cron will just ignore
  the PR and it's no skin off our back), so `_trigger` the cron for
  validation.
- Also do the same when skipchecks is set as it's very similar.

In reality we might want to hook off of the state transitioning to
reviewed but I'm not sure there's good ways to do that (triggering a
cron inside a compute doesn't seem like a good idea).

Update a pair of tests which would approve a multi-commit PR without
setting a merge method, just because the helper they use to build the
PR happens to create multiple commits.

Fix #891
---
 runbot_merge/models/pull_requests.py | 19 ++++++++++++-------
 runbot_merge/tests/test_basic.py     | 10 ++++++----
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 8a1bd658..e4676bdb 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -777,6 +777,9 @@ class PullRequests(models.Model):
                 case commands.SkipChecks() if is_admin:
                     self.batch_id.skipchecks = True
                     self.reviewed_by = author
+                    if not (self.squash or self.merge_method):
+                        self.env.ref('runbot_merge.check_linked_prs_status')._trigger()
+
                     for p in self.batch_id.prs - self:
                         if not p.reviewed_by:
                             p.reviewed_by = author
@@ -999,13 +1002,12 @@ class PullRequests(models.Model):
     def _approve(self, author, login):
         oldstate = self.state
         newstate = RPLUS.get(self.state)
-        msg = None
         if not author.email:
-            msg = "I must know your email before you can review PRs. Please contact an administrator."
+            return "I must know your email before you can review PRs. Please contact an administrator."
         elif not newstate:
-            msg = "this PR is already reviewed, reviewing it again is useless."
-        else:
-            self.reviewed_by = author
+            return "this PR is already reviewed, reviewing it again is useless."
+
+        self.reviewed_by = author
         _logger.debug(
             "r+ on %s by %s (%s->%s) status=%s message? %s",
             self.display_name, author.github_login,
@@ -1020,7 +1022,9 @@ class PullRequests(models.Model):
                 pull_request=self.number,
                 format_args={'user': login, 'pr': self},
             )
-        return msg
+        if not (self.squash or self.merge_method):
+            self.env.ref('runbot_merge.check_linked_prs_status')._trigger()
+        return None
 
     def message_post(self, **kw):
         if author := self.env.cr.precommit.data.get('change-author'):
@@ -1362,7 +1366,8 @@ class PullRequests(models.Model):
             if pair[0] != 'squash'
         )
         for r in self.search([
-            ('state', '=', 'ready'),
+            ('state', 'in', ("approved", "ready")),
+            ('staging_id', '=', False),
             ('squash', '=', False),
             ('merge_method', '=', False),
             ('method_warned', '=', False),
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 6dec9506..40b26eda 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -1050,7 +1050,7 @@ def test_ci_failure_after_review(env, repo, users, config):
     """
     with repo:
         prx = _simple_init(repo)
-        prx.post_comment('hansen r+', config['role_reviewer']['token'])
+        prx.post_comment('hansen r+ rebase-ff', config['role_reviewer']['token'])
     env.run_crons()
 
     for ctx, url in [
@@ -1066,8 +1066,9 @@ def test_ci_failure_after_review(env, repo, users, config):
         env.run_crons()
 
     assert prx.comments == [
-        (users['reviewer'], 'hansen r+'),
+        (users['reviewer'], 'hansen r+ rebase-ff'),
         seen(env, prx, users),
+        (users['user'], "Merge method set to rebase and fast-forward."),
         (users['user'], "@{user} @{reviewer} 'ci/runbot' failed on this reviewed PR.".format_map(users)),
         (users['user'], "@{user} @{reviewer} 'legal/cla' failed on this reviewed PR.".format_map(users)),
         (users['user'], "@{user} @{reviewer} 'legal/cla' failed on this reviewed PR.".format_map(users)),
@@ -1302,14 +1303,15 @@ class TestRetry:
         """
         with repo:
             prx = _simple_init(repo)
-            prx.post_comment('hansen r+', config['role_reviewer']['token'])
+            prx.post_comment('hansen r+ rebase-ff', config['role_reviewer']['token'])
             prx.post_comment('hansen retry', config['role_reviewer']['token'])
         env.run_crons()
 
         assert prx.comments == [
-            (users['reviewer'], 'hansen r+'),
+            (users['reviewer'], 'hansen r+ rebase-ff'),
             (users['reviewer'], 'hansen retry'),
             seen(env, prx, users),
+            (users['user'], "Merge method set to rebase and fast-forward."),
             (users['user'], "@{reviewer} retry makes no sense when the PR is not in error.".format_map(users)),
         ]
 

From 20d259aa7749de9511f85a57298f4d08c3819fa1 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 20 Jun 2024 13:49:17 +0200
Subject: [PATCH 134/192] [IMP] runbot_merge: always display PR title

Previous version would always hide the title if the PR was
blocked (e.g. blocked or failed), turns out there are people who
actually use the PR title on the main dashboard, so suppressing that
is inconvenient for them.

Try to show the PR title if available, and add the blocked message if
present.
---
 runbot_merge/views/templates.xml | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index cdad7aa3..4bb20542 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -7,11 +7,19 @@
     <template id="link-pr" name="create a link to `pr`">
         <t t-set="title">
             <t t-if="pr.repository.group_id &lt;= env.user.groups_id">
-                <t t-esc="pr.message.split('\n')[0]"/>
+                <t t-out="pr.message.split('\n', 1)[0]"/>
             </t>
         </t>
+        <t t-set="title">
+          <t t-if="title.strip() and pr.blocked" >
+            <t t-out="title.strip()"/>: <t t-out="pr.blocked"/>
+          </t>
+          <t t-else="">
+            <t t-out="pr.blocked or title.strip()"/>
+          </t>
+        </t>
         <a t-attf-href="https://github.com/{{ pr.repository.name }}/pull/{{ pr.number }}"
-           t-att-title="pr.blocked or title.strip()"
+           t-att-title="title"
            t-att-target="target or None"
            t-att-class="classes or None"
         ><t t-esc="pr.display_name"/></a>

From 4a521e12512604a1edae4ecdf5f044940c82af24 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 20 Jun 2024 14:21:40 +0200
Subject: [PATCH 135/192] [IMP] runbot_merge: hide backend links from
 group_user

The backend links in the PR dashboard were gated behind the
`group_user` (internal user) group, however turns out while internal
users have read access to PRs they don't have access to ancillary
objects (e.g. batches, stagings, the link between stagings and
batches), and I think the only way to fix the issue would be to move
it to an optional inheritance (inheritance + group), because `groups`
on view nodes only hides the content without removing it.

I believe in more recent Odoo versions this actually works correctly,
so that might actually be more of an incentive to upgrade...
---
 runbot_merge/views/templates.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index 4bb20542..62b9069c 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -430,7 +430,7 @@
                     </a>
                     <a t-attf-href="/web#view_type=form&amp;model=runbot_merge.pull_requests&amp;id={{pr.id}}"
                         class="btn btn-sm btn-secondary align-top float-right"
-                        groups="base.group_user">View in backend</a>
+                        groups="runbot_merge.group_admin">View in backend</a>
                 </h1>
                 <h6>Created by <span t-field="pr.author.display_name"/></h6>
                 <t t-set="tmpl">
@@ -597,7 +597,7 @@ action = batches
                                    title="Open on Github"
                                    t-att-href="p['github_url']"
                                 />
-                                <a groups="base.group_user"
+                                <a groups="runbot_merge.group_admin"
                                    title="Open in Backend"
                                    t-attf-class="fa fa-external-link"
                                    t-att-href="p['backend_url']"

From f30367443409d2d34c15a959dba4e8288aa09ecf Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 21 Jun 2024 10:27:01 +0200
Subject: [PATCH 136/192] [FIX] *: re-enable notification on status failure

If a PR gets approved *then* fails CI, there should be a notification
warning the author & reviewer since
48e08b657b23760e6d45f397cf59e7710bc7f89a, it even has a test, which
passes (in fact it has *two*, one of which is redundant, so merge
`test_ci_failure_after_review` into the later `test_ci_approved`).

*However* this is in runbot_merge, turns out in
fafa7ef437d75d72e34be91af72e16c52135b7fa some refactoring was done in
order to override the notification and customise it for *forward
ports* with a failed status... except that override never called its
`super()`, so as soon as forwardport is installed the base
notification stops working, and that's been that since October
2019 (had been added in March that year, ignoring deployment lag).

This can be revealed by adding the corresponding check in the
*forwardport* tests, revealing the failure.

This was a pain to track down, thankfully it reproduced relatively
easily locally.

While this could be resolved in the override, might as well fold it
into the base method in furtherance of #789: the mergebot is only
used by odoo, and only with both modules combined, so splitting them
is not useful. And furthermore it things should work fine with the
forwardport installed but unused.

Fixes #894
---
 forwardport/models/project.py        | 12 -----
 forwardport/tests/test_updates.py    | 15 +++++-
 runbot_merge/models/pull_requests.py |  8 +++
 runbot_merge/tests/test_basic.py     | 74 +++++++++-------------------
 4 files changed, 45 insertions(+), 64 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 892c2d58..abca0e7d 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -275,18 +275,6 @@ class PullRequests(models.Model):
             })
         return r
 
-    def _notify_ci_failed(self, ci):
-        # only care about FP PRs which are not staged / merged yet
-        # NB: probably ignore approved PRs as normal message will handle them?
-        if not (self.state == 'opened' and self.parent_id):
-            return
-
-        self.env.ref('runbot_merge.forwardport.ci.failed')._send(
-            repository=self.repository,
-            pull_request=self.number,
-            token_field='fp_github_token',
-            format_args={'pr': self, 'ci': ci},
-        )
 
     def _validate(self, statuses):
         failed = super()._validate(statuses)
diff --git a/forwardport/tests/test_updates.py b/forwardport/tests/test_updates.py
index 0ee884d2..d68c6c92 100644
--- a/forwardport/tests/test_updates.py
+++ b/forwardport/tests/test_updates.py
@@ -33,11 +33,22 @@ def test_update_pr(env, config, make_repo, users, merge_parent) -> None:
             ref='heads/hugechange'
         )
         pr = prod.make_pr(target='a', head='hugechange')
-        prod.post_status(p_1, 'success', 'legal/cla')
-        prod.post_status(p_1, 'success', 'ci/runbot')
         pr.post_comment('hansen r+', config['role_reviewer']['token'])
 
+        prod.post_status(p_1, 'success', 'legal/cla')
+        prod.post_status(p_1, 'failure', 'ci/runbot')
     env.run_crons()
+
+    assert pr.comments == [
+        (users['reviewer'], 'hansen r+'),
+        seen(env, pr, users),
+        (users['user'], "@{user} @{reviewer} 'ci/runbot' failed on this reviewed PR.".format_map(users)),
+    ]
+
+    with prod:
+        prod.post_status(p_1, 'success', 'ci/runbot')
+    env.run_crons()
+
     with prod:
         prod.post_status('staging.a', 'success', 'legal/cla')
         prod.post_status('staging.a', 'success', 'ci/runbot')
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index e4676bdb..983ce2f1 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1168,6 +1168,14 @@ class PullRequests(models.Model):
                 pull_request=self.number,
                 format_args={'pr': self, 'status': ci}
             )
+        elif self.state == 'opened' and self.parent_id:
+            # only care about FP PRs which are not approved / staged / merged yet
+            self.env.ref('runbot_merge.forwardport.ci.failed')._send(
+                repository=self.repository,
+                pull_request=self.number,
+                token_field='fp_github_token',
+                format_args={'pr': self, 'ci': ci},
+            )
 
     def _auto_init(self):
         for field in self._fields.values():
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 40b26eda..0bf57dd3 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -1042,38 +1042,6 @@ def test_rebase_failure(env, repo, users, config):
         'b': 'b',
     }
 
-def test_ci_failure_after_review(env, repo, users, config):
-    """ If a PR is r+'d but the CI ends up failing afterwards, ping the user
-    so they're aware. This is useful for the more "fire and forget" approach
-    especially small / simple PRs where you assume they're going to pass and
-    just r+ immediately.
-    """
-    with repo:
-        prx = _simple_init(repo)
-        prx.post_comment('hansen r+ rebase-ff', config['role_reviewer']['token'])
-    env.run_crons()
-
-    for ctx, url in [
-        ('ci/runbot', 'https://a'),
-        ('ci/runbot', 'https://a'),
-        ('legal/cla', 'https://b'),
-        ('foo/bar', 'https://c'),
-        ('ci/runbot', 'https://a'),
-        ('legal/cla', 'https://d'), # url changes so different from the previous
-    ]:
-        with repo:
-            repo.post_status(prx.head, 'failure', ctx, target_url=url)
-        env.run_crons()
-
-    assert prx.comments == [
-        (users['reviewer'], 'hansen r+ rebase-ff'),
-        seen(env, prx, users),
-        (users['user'], "Merge method set to rebase and fast-forward."),
-        (users['user'], "@{user} @{reviewer} 'ci/runbot' failed on this reviewed PR.".format_map(users)),
-        (users['user'], "@{user} @{reviewer} 'legal/cla' failed on this reviewed PR.".format_map(users)),
-        (users['user'], "@{user} @{reviewer} 'legal/cla' failed on this reviewed PR.".format_map(users)),
-    ]
-
 def test_reopen_merged_pr(env, repo, config, users):
     """ Reopening a *merged* PR should cause us to immediately close it again,
     and insult whoever did it
@@ -3902,28 +3870,34 @@ class TestFeedback:
     def test_ci_approved(self, repo, env, users, config):
         """CI failing on an r+'d PR sends feedback"""
         with repo:
-            m = repo.make_commit(None, 'initial', None, tree={'m': 'm'})
-            repo.make_ref('heads/master', m)
+            [m] = repo.make_commits(None, Commit('initial', tree={'m': 'm'}), ref="heads/master")
 
-            c1 = repo.make_commit(m, 'first', None, tree={'m': 'c1'})
-            prx = repo.make_pr(title='title', body='body', target='master', head=c1)
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number)
-        ])
-
-        with repo:
-            prx.post_comment('hansen r+', config['role_reviewer']['token'])
-        assert pr.state == 'approved'
-
-        with repo:
-            repo.post_status(prx.head, 'failure', 'ci/runbot')
+            [c1] = repo.make_commits(m, Commit('first', tree={'m': 'c1'}))
+            pr = repo.make_pr(title='title', body='body', target='master', head=c1)
+            pr.post_comment('hansen r+', config['role_reviewer']['token'])
         env.run_crons()
 
-        assert prx.comments == [
+        pr_id = to_pr(env, pr)
+        assert pr_id.state == 'approved'
+
+        for ctx, url in [
+            ('ci/runbot', 'https://a'),
+            ('ci/runbot', 'https://a'),
+            ('legal/cla', 'https://b'),
+            ('foo/bar', 'https://c'),
+            ('ci/runbot', 'https://a'),
+            ('legal/cla', 'https://d'),  # url changes so different from the previous
+        ]:
+            with repo:
+                repo.post_status(pr_id.head, 'failure', ctx, target_url=url)
+            env.run_crons()
+
+        assert pr.comments == [
             (users['reviewer'], 'hansen r+'),
-            seen(env, prx, users),
-            (users['user'], "@%(user)s @%(reviewer)s 'ci/runbot' failed on this reviewed PR." % users)
+            seen(env, pr, users),
+            (users['user'], "@{user} @{reviewer} 'ci/runbot' failed on this reviewed PR.".format_map(users)),
+            (users['user'], "@{user} @{reviewer} 'legal/cla' failed on this reviewed PR.".format_map(users)),
+            (users['user'], "@{user} @{reviewer} 'legal/cla' failed on this reviewed PR.".format_map(users)),
         ]
 
     def test_review_failed(self, repo, env, users, config):

From 737cbd5de240550f4f6b5a2e3fafcf987ff9d97f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 21 Jun 2024 10:40:06 +0200
Subject: [PATCH 137/192] [IMP] *: merge fw overrides into their parent

Not actually useful in any way, but it does remove a few lines, avoids
a few dupe writes, and furthers the cause of #789
---
 forwardport/models/project.py        | 19 -------------------
 runbot_merge/models/pull_requests.py | 15 ++++++++++++---
 2 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index abca0e7d..b0d743e8 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -262,25 +262,6 @@ class PullRequests(models.Model):
                     )
         return r
 
-    def _try_closing(self, by):
-        r = super()._try_closing(by)
-        if r:
-            self.with_context(forwardport_detach_warn=False).write({
-                'parent_id': False,
-                'detach_reason': f"Closed by {by}",
-            })
-            self.search([('parent_id', '=', self.id)]).write({
-                'parent_id': False,
-                'detach_reason': f"{by} closed parent PR {self.display_name}",
-            })
-        return r
-
-
-    def _validate(self, statuses):
-        failed = super()._validate(statuses)
-        self.batch_id._schedule_fp_followup()
-        return failed
-
     def _commits_lazy(self):
         s = requests.Session()
         s.headers['Authorization'] = 'token %s' % self.repository.project_id.fp_github_token
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 983ce2f1..9afae63a 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1067,6 +1067,7 @@ class PullRequests(models.Model):
                     status = statuses.get(ci) or {'state': 'pending'}
                     if status['state'] in ('error', 'failure'):
                         pr._notify_ci_new_failure(ci, status)
+        self.batch_id._schedule_fp_followup()
 
     def modified(self, fnames, create=False, before=False):
         """ By default, Odoo can't express recursive *dependencies* which is
@@ -1443,12 +1444,20 @@ class PullRequests(models.Model):
         WHERE id = %s AND state != 'merged' AND state != 'closed'
         FOR UPDATE SKIP LOCKED;
         ''', [self.id])
-        r = self.env.cr.fetchone()
-        if not r:
+        if not self.env.cr.rowcount:
             return False
 
         self.unstage("closed by %s", by)
-        self.write({'closed': True, 'reviewed_by': False})
+        self.with_context(forwardport_detach_warn=False).write({
+            'closed': True,
+            'reviewed_by': False,
+            'parent_id': False,
+            'detach_reason': f"Closed by {by}",
+        })
+        self.search([('parent_id', '=', self.id)]).write({
+            'parent_id': False,
+            'detach_reason': f"{by} closed parent PR {self.display_name}",
+        })
 
         return True
 

From 3410f50248723bb2993ab71bb5df9fa31424a4c3 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 21 Jun 2024 10:41:01 +0200
Subject: [PATCH 138/192] [FIX] runbot_merge: `Commit.create`

The method was not marked as a create, following which it did not
allow creating commits via the UI (annoying for testing / reproducing
issues involving statuses).
---
 runbot_merge/models/pull_requests.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 9afae63a..260e0c9b 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1776,6 +1776,7 @@ class Commit(models.Model):
     commit_ids = fields.Many2many('runbot_merge.stagings', relation='runbot_merge_stagings_commits', column2='staging_id', column1='commit_id')
     pull_requests = fields.One2many('runbot_merge.pull_requests', compute='_compute_prs')
 
+    @api.model_create_single
     def create(self, values):
         values['to_check'] = True
         r = super(Commit, self).create(values)

From 906505ed15fa114e7047cfdef4ca3bc365cf0201 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 21 Jun 2024 10:42:08 +0200
Subject: [PATCH 139/192] [IMP] runbot_merge: filter on the base attribute not
 computed

Should not actually do anything relevant, but seems like a good idea.
---
 runbot_merge/models/pull_requests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 260e0c9b..f657011e 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1058,7 +1058,7 @@ class PullRequests(models.Model):
         # could have two PRs (e.g. one open and one closed) at least
         # temporarily on the same head, or on the same head with different
         # targets
-        updateable = self.filtered(lambda p: p.state != 'merged')
+        updateable = self.filtered(lambda p: not p.merge_date)
         updateable.statuses = statuses
         for pr in updateable:
             if pr.status == "failure":

From 92e8eecbb54a33329da991501b457cb11d66bbb9 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 21 Jun 2024 10:42:37 +0200
Subject: [PATCH 140/192] [FIX] runbot_merge: ability to create PRs via the UI

This is useful to repro issues.

60c4b5141d55ebb73b5f8e4b1f1738c80369722e added `inverse=readonly`
hooks to various newly computed fields to ensure they can not be *written*
to, either overwriting the content (stored) or silently being
dropped (non-stored).

However because they're `inverse` hooks this had the effect of making
them writeable from the backend UI since the ORM uses `inverse` as a
signal to make the field writeable. This then caused the web client to
send stuff for those fields, which are not necessarily even visible in
the form, leading to write errors when trying to save a PR creation.

By marking the fields as `readonly` explicitly we make sure that
doesn't happen, and we can create PRs from the backend UI (kinda, I
think the label is still an issue).
---
 runbot_merge/models/pull_requests.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index f657011e..5685721e 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -335,6 +335,7 @@ class PullRequests(models.Model):
     merge_date = fields.Datetime(
         related='batch_id.merge_date',
         inverse=readonly,
+        readonly=True,
         tracking=True,
         store=True,
     )
@@ -383,7 +384,7 @@ class PullRequests(models.Model):
 
     reviewed_by = fields.Many2one('res.partner', index=True, tracking=True)
     delegates = fields.Many2many('res.partner', help="Delegate reviewers, not intrinsically reviewers but can review this PR")
-    priority = fields.Selection(related="batch_id.priority", inverse=readonly)
+    priority = fields.Selection(related="batch_id.priority", inverse=readonly, readonly=True)
 
     overrides = fields.Char(required=True, default='{}', tracking=True)
     statuses = fields.Text(help="Copy of the statuses from the HEAD commit, as a Python literal", default="{}")
@@ -396,12 +397,12 @@ class PullRequests(models.Model):
         ('pending', 'Pending'),
         ('failure', 'Failure'),
         ('success', 'Success'),
-    ], compute='_compute_statuses', store=True, inverse=readonly, column_type=enum(_name, 'status'))
+    ], compute='_compute_statuses', store=True, inverse=readonly, readonly=True, column_type=enum(_name, 'status'))
     previous_failure = fields.Char(default='{}')
 
     batch_id = fields.Many2one('runbot_merge.batch', index=True)
-    staging_id = fields.Many2one('runbot_merge.stagings', compute='_compute_staging', inverse=readonly, store=True)
-    staging_ids = fields.Many2many('runbot_merge.stagings', string="Stagings", compute='_compute_stagings', inverse=readonly, context={"active_test": False})
+    staging_id = fields.Many2one('runbot_merge.stagings', compute='_compute_staging', inverse=readonly, readonly=True, store=True)
+    staging_ids = fields.Many2many('runbot_merge.stagings', string="Stagings", compute='_compute_stagings', inverse=readonly, readonly=True, context={"active_test": False})
 
     @api.depends('batch_id.batch_staging_ids.runbot_merge_stagings_id.active')
     def _compute_staging(self):

From 7cd9afe7f2ff2cbaaabe0ecf6eb364b55aaf8d60 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 21 Jun 2024 10:58:05 +0200
Subject: [PATCH 141/192] [IMP] runbot_merge: trigger commits cron

The commit cron needs to be triggered any time we:

- create a new commit
- update a commit to set its `to_check`

So do that in create and write as well as the SQL query in the
webhook handler.

This should mean we don't need the periodic cron anymore, but for
safety's sake run it on 30mn for now.

TBF even if we miss triggers, the next `status` webhook hitting will
check all the relevant commits anyway...
---
 runbot_merge/controllers/__init__.py | 1 +
 runbot_merge/data/merge_cron.xml     | 2 +-
 runbot_merge/models/pull_requests.py | 3 +++
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index bdbd96c8..86d65541 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -369,6 +369,7 @@ def handle_status(env, event):
                 statuses = c.statuses::jsonb || EXCLUDED.statuses::jsonb
             WHERE NOT c.statuses::jsonb @> EXCLUDED.statuses::jsonb
     """, [event['sha'], status_value])
+    env.ref("runbot_merge.process_updated_commits")._trigger()
 
     return 'ok'
 
diff --git a/runbot_merge/data/merge_cron.xml b/runbot_merge/data/merge_cron.xml
index f8736785..b0ff6e1d 100644
--- a/runbot_merge/data/merge_cron.xml
+++ b/runbot_merge/data/merge_cron.xml
@@ -64,7 +64,7 @@
     <field name="model_id" ref="model_runbot_merge_commit"/>
     <field name="state">code</field>
     <field name="code">model._notify()</field>
-    <field name="interval_number">1</field>
+    <field name="interval_number">30</field>
     <field name="interval_type">minutes</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 5685721e..9b57924b 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1781,11 +1781,14 @@ class Commit(models.Model):
     def create(self, values):
         values['to_check'] = True
         r = super(Commit, self).create(values)
+        self.env.ref("runbot_merge.process_updated_commits")._trigger()
         return r
 
     def write(self, values):
         values.setdefault('to_check', True)
         r = super(Commit, self).write(values)
+        if values['to_check']:
+            self.env.ref("runbot_merge.process_updated_commits")._trigger()
         return r
 
     def _notify(self):

From b109225f44d0d9548e08637203dd4354a2d4837c Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 21 Jun 2024 14:38:47 +0200
Subject: [PATCH 142/192] [IMP] runbot_merge: quality of feedback on errorneous
 commands

- When a redundant approval is sent to a PR, notify but don't ignore
  the entire command set, there's no actual risk.
- Indicate that the entire comment was ignored when finding something
  which does not parse.

Fixes #892, fixes #893
---
 forwardport/tests/test_limit.py      |  4 ++--
 runbot_merge/models/commands.py      |  2 +-
 runbot_merge/models/pull_requests.py | 18 +++++++++++++-----
 runbot_merge/tests/test_basic.py     |  6 +++---
 4 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index 14645c26..5e59c008 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -129,15 +129,15 @@ def test_disable(env, config, make_repo, users):
     # use a set because git webhooks delays might lead to mis-ordered
     # responses and we don't care that much
     assert set(pr.comments) == {
+        seen(env, pr, users),
         (users['reviewer'], "hansen r+ up to"),
-        (users['user'], "@{reviewer} please provide a branch to forward-port to.".format_map(users)),
+        (users['user'], "@{reviewer} please provide a branch to forward-port to.\n\nFor your own safety I've ignored *everything in your entire comment*.".format_map(users)),
         (users['reviewer'], "hansen up to b"),
         (users['user'], "@{reviewer} branch 'b' is disabled, it can't be used as a forward port target.".format_map(users)),
         (users['reviewer'], "hansen up to foo"),
         (users['user'], "@{reviewer} there is no branch 'foo', it can't be used as a forward port target.".format_map(users)),
         (users['reviewer'], "hansen up to c"),
         (users['user'], "Forward-porting to 'c'."),
-        seen(env, pr, users),
     }
 
 
diff --git a/runbot_merge/models/commands.py b/runbot_merge/models/commands.py
index 2911638b..1ebc5fb6 100644
--- a/runbot_merge/models/commands.py
+++ b/runbot_merge/models/commands.py
@@ -305,7 +305,7 @@ class Parser:
         if limit := next(self.it, None):
             return Limit(limit)
         else:
-            raise CommandError("please provide a branch to forward-port to.")
+            raise CommandError("please provide a branch to forward-port to")
 
     def parse_close(self) -> Close:
         return Close()
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 9b57924b..d090eef4 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -656,7 +656,7 @@ class PullRequests(models.Model):
                 utils.shorten(comment['body'] or '', 50),
                 exc_info=True
             )
-            feedback(message=f"@{login} {e.args[0]}")
+            feedback(message=f"@{login} {e.args[0]}.\n\nFor your own safety I've ignored *everything in your entire comment*.")
             return 'error'
 
         is_admin, is_reviewer, is_author = self._pr_acl(author)
@@ -1002,17 +1002,25 @@ class PullRequests(models.Model):
 
     def _approve(self, author, login):
         oldstate = self.state
-        newstate = RPLUS.get(self.state)
+        newstate = RPLUS.get(oldstate)
         if not author.email:
             return "I must know your email before you can review PRs. Please contact an administrator."
-        elif not newstate:
-            return "this PR is already reviewed, reviewing it again is useless."
+
+        if not newstate:
+            # Don't fail the entire command if someone tries to approve an
+            # already-approved PR.
+            self.env['runbot_merge.pull_requests.feedback'].create({
+                'repository': self.repository.id,
+                'pull_request': self.number,
+                'message': "This PR is already reviewed, reviewing it again is useless.",
+            })
+            return None
 
         self.reviewed_by = author
         _logger.debug(
             "r+ on %s by %s (%s->%s) status=%s message? %s",
             self.display_name, author.github_login,
-            oldstate, newstate or oldstate,
+            oldstate, newstate,
             self.status, self.status == 'failure'
         )
         if self.status == 'failure':
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 0bf57dd3..79ce42e2 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -3105,7 +3105,7 @@ class TestReviewing:
             (users['user'], "I'm sorry, @{}. I'm afraid I can't do that.".format(users['other'])),
             (users['reviewer'], 'hansen r+'),
             (users['reviewer'], 'hansen r+'),
-            (users['user'], "@{} this PR is already reviewed, reviewing it again is useless.".format(
+            (users['user'], "This PR is already reviewed, reviewing it again is useless.".format(
                  users['reviewer'])),
         ]
 
@@ -3613,8 +3613,8 @@ class TestRecognizeCommands:
             (users['reviewer'], "hansen do the thing"),
             (users['reviewer'], "hansen @bobby-b r+ :+1:"),
             seen(env, pr, users),
-            (users['user'], "@{reviewer} unknown command 'do'".format_map(users)),
-            (users['user'], "@{reviewer} unknown command '@bobby-b'".format_map(users)),
+            (users['user'], "@{reviewer} unknown command 'do'.\n\nFor your own safety I've ignored *everything in your entire comment*.".format_map(users)),
+            (users['user'], "@{reviewer} unknown command '@bobby-b'.\n\nFor your own safety I've ignored *everything in your entire comment*.".format_map(users)),
         ]
 
 class TestRMinus:

From f3a0a5c27c62426a69dd29f14af85fb9f1b75c1a Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 21 Jun 2024 16:33:44 +0200
Subject: [PATCH 143/192] [FIX] runbot_merge: tracking message author on
 PullRequest events

d4fa1fd35315d330566e37f515a937f722859ef7 added tracking to changes
from *comments* (as well as a few hacks around authorship transfer),
however it missed two things:

First, it set the `change-author` during comments handling only, so
changes from the `PullRequest` hook e.g. open, synchronise, close,
edit, don't get attributed to their actual source, and instead just
fall back to uid(1). This is easy enough to fix as the `sender` is
always provided, that can be resolved to a partner which is then set
as the author of whatever changes happen.

Second, I actually missed one of the message hooks: there's both
`_message_log` and `_message_log_batch` and they don't call one
another, so both have to be overridden in order for tracking to be
consistent. In this case, specifically, the *creation* of a tracked
object goes through `_message_log_batch` (since that's a very generic
message and so works on every tracked object created during the
transaction... even though batch has a message per record anyway...)
while *updates* go through `_message_log`.

Fixes #895
---
 conftest.py                          |  8 ++++++++
 runbot_merge/controllers/__init__.py |  5 +++++
 runbot_merge/models/pull_requests.py |  7 +++++++
 runbot_merge/tests/test_basic.py     | 19 ++++++++++++++-----
 4 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/conftest.py b/conftest.py
index 21af5274..28980399 100644
--- a/conftest.py
+++ b/conftest.py
@@ -160,6 +160,14 @@ def rolemap(request, config):
 
 @pytest.fixture
 def partners(env, config, rolemap):
+    """This specifically does not create partners for ``user`` and ``other``
+    so they can be generated on-interaction, as "external" users.
+
+    The two differ in that ``user`` has ownership of the org and can manage
+    repos there, ``other`` is completely unrelated to anything so useful to
+    check for interaction where the author only has read access to the reference
+    repositories.
+    """
     m = {}
     for role, u in rolemap.items():
         if role in ('user', 'other'):
diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 86d65541..278b6609 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -254,6 +254,11 @@ def handle_pr(env, event):
         headers.get('X-Github-Delivery'),
         headers.get('User-Agent'),
      )
+    sender = env['res.partner'].search([('github_login', '=', event['sender']['login'])], limit=1)
+    if not sender:
+        sender = env['res.partner'].create({'name': event['sender']['login'], 'github_login': event['sender']['login']})
+    env.cr.precommit.data['change-author'] = sender.id
+
     if event['action'] == 'opened':
         author_name = pr['user']['login']
         author = env['res.partner'].search([('github_login', '=', author_name)], limit=1)
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index d090eef4..9adb5ae0 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1049,6 +1049,13 @@ class PullRequests(models.Model):
             kw['body'] = html_escape(message)
         return super()._message_log(**kw)
 
+    def _message_log_batch(self, **kw):
+        if author := self.env.cr.precommit.data.get('change-author'):
+            kw['author_id'] = author
+        if message := self.env.cr.precommit.data.get('change-message'):
+            kw['bodies'] = {p.id: html_escape(message) for p in self}
+        return super()._message_log_batch(**kw)
+
     def _pr_acl(self, user):
         if not self:
             return ACL(False, False, False)
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 79ce42e2..d64596e6 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -65,6 +65,13 @@ def test_trivial_flow(env, repo, page, users, config):
         )
         pr = repo.make_pr(title="gibberish", body="blahblah", target='master', head='other')
 
+        [c2] = repo.make_commits(
+            'other',
+            Commit('forgot a bit', tree={'whee': 'kjfdsh'}),
+            ref='heads/other',
+            make=False,
+        )
+
     pr_id = to_pr(env, pr)
     assert pr_id.state == 'opened'
     env.run_crons()
@@ -79,12 +86,12 @@ def test_trivial_flow(env, repo, page, users, config):
         [e.text_content() for e in pr_dashboard.cssselect('dl.runbot-merge-fields dd')],
     )) == {
         'label': f"{config['github']['owner']}:other",
-        'head': c1,
+        'head': c2,
     }
 
     with repo:
-        repo.post_status(c1, 'success', 'legal/cla')
-        repo.post_status(c1, 'success', 'ci/runbot')
+        repo.post_status(c2, 'success', 'legal/cla')
+        repo.post_status(c2, 'success', 'ci/runbot')
     env.run_crons()
     assert pr_id.state == 'validated'
 
@@ -140,6 +147,7 @@ def test_trivial_flow(env, repo, page, users, config):
     assert repo.read_tree(master) == {
         'a': 'some other content',
         'b': 'a second file',
+        'whee': 'kjfdsh',
     }
     assert master.message == "gibberish\n\nblahblah\n\ncloses {repo.name}#1"\
                              "\n\nSigned-off-by: {reviewer.formatted_email}"\
@@ -156,8 +164,9 @@ def test_trivial_flow(env, repo, page, users, config):
     ])
 
     assert list(messages) == [
-        ('OdooBot', '<p>Pull Request created</p>', []),
-        ('OdooBot', f'<p>statuses changed on {c1}</p>', [('Opened', 'Validated')]),
+        (users['user'], '<p>Pull Request created</p>', []),
+        (users['user'], '', [(c1, c2)]),
+        ('OdooBot', f'<p>statuses changed on {c2}</p>', [('Opened', 'Validated')]),
         # reviewer approved changing the state and setting reviewer as reviewer
         # plus set merge method
         ('Reviewer', '', [

From dc90a207d6634774a7eecc3baaff541cd200a955 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 24 Jun 2024 22:16:43 +0200
Subject: [PATCH 144/192] [ADD] runbot_merge: help command, and help on error

Fixes #898
---
 conftest.py                          |  11 +++
 forwardport/tests/test_limit.py      |  31 +++++++-
 runbot_merge/models/commands.py      |  75 ++++++++++++++++++-
 runbot_merge/models/pull_requests.py |  63 +++++++++++++++-
 runbot_merge/tests/test_basic.py     |  66 ++++++++++++++++-
 runbot_merge/tests/test_multirepo.py |   2 +
 runbot_merge/tests/test_oddities.py  | 105 ++++++++++++++++++++++++++-
 7 files changed, 342 insertions(+), 11 deletions(-)

diff --git a/conftest.py b/conftest.py
index 28980399..9572cea3 100644
--- a/conftest.py
+++ b/conftest.py
@@ -481,6 +481,17 @@ def env(request, port, server, db, default_crons):
         if b"Traceback (most recent call last):" in server[1]:
             pytest.fail("unexpected error in logs, fix, or mark function as `expect_log_errors` to require.")
 
+@pytest.fixture
+def reviewer_admin(env, partners):
+    env['res.users'].create({
+        'partner_id': partners['reviewer'].id,
+        'login': 'reviewer',
+        'groups_id': [
+            (4, env.ref("base.group_user").id, 0),
+            (4, env.ref("runbot_merge.group_admin").id, 0),
+        ],
+    })
+
 def check(response):
     assert response.ok, response.text or response.reason
     return response
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index 5e59c008..3e7bb9ed 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -131,7 +131,36 @@ def test_disable(env, config, make_repo, users):
     assert set(pr.comments) == {
         seen(env, pr, users),
         (users['reviewer'], "hansen r+ up to"),
-        (users['user'], "@{reviewer} please provide a branch to forward-port to.\n\nFor your own safety I've ignored *everything in your entire comment*.".format_map(users)),
+        (users['user'], """\
+@{reviewer} please provide a branch to forward-port to.
+
+For your own safety I've ignored *everything in your entire comment*.
+
+Currently available commands:
+
+|command||
+|-|-|
+|`help`|displays this help|
+|`r(eview)+`|approves the PR, if it's a forwardport also approves all non-detached parents|
+|`r(eview)=<number>`|only approves the specified parents|
+|`fw=no`|does not forward-port this PR|
+|`fw=default`|forward-ports this PR normally|
+|`fw=skipci`|does not wait for a forward-port's statuses to succeed before creating the next one|
+|`up to <branch>`|only ports this PR forward to the specified branch (included)|
+|`merge`|integrate the PR with a simple merge commit, using the PR description as message|
+|`rebase-merge`|rebases the PR on top of the target branch the integrates with a merge commit, using the PR description as message|
+|`rebase-ff`|rebases the PR on top of the target branch, then fast-forwards|
+|`squash`|squashes the PR as a single commit on the target branch, using the PR description as message|
+|`delegate+`|grants approval rights to the PR author|
+|`delegate=<...>`|grants approval rights on this PR to the specified github users|
+|`default`|stages the PR normally|
+|`priority`|tries to stage this PR first, then adds `default` PRs if the staging has room|
+|`alone`|stages this PR only with other PRs of the same priority|
+|`cancel=staging`|automatically cancels the current staging when this PR becomes ready|
+|`check`|fetches or refreshes PR metadata, resets mergebot state|
+
+Note: this help text is dynamic and will change with the state of the PR.
+""".format_map(users)),
         (users['reviewer'], "hansen up to b"),
         (users['user'], "@{reviewer} branch 'b' is disabled, it can't be used as a forward port target.".format_map(users)),
         (users['reviewer'], "hansen up to foo"),
diff --git a/runbot_merge/models/commands.py b/runbot_merge/models/commands.py
index 1ebc5fb6..684209e3 100644
--- a/runbot_merge/models/commands.py
+++ b/runbot_merge/models/commands.py
@@ -3,7 +3,7 @@ from collections.abc import Iterator
 from dataclasses import dataclass, field
 from functools import partial
 from operator import contains
-from typing import Callable, List, Optional, Union
+from typing import Callable, List, Optional, Union, Tuple
 
 
 def tokenize(line: str) -> Iterator[str]:
@@ -72,11 +72,18 @@ class Approve:
             return f"r={ids}"
         return 'review+'
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield "r(eview)+", "approves the PR, if it's a forwardport also approves all non-detached parents"
+        yield "r(eview)=<number>", "only approves the specified parents"
 
 class Reject:
     def __str__(self) -> str:
         return 'review-'
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield "r(eview)-", "removes approval of a previously approved PR, if the PR is staged the staging will be cancelled"
 
 class MergeMethod(enum.Enum):
     SQUASH = 'squash'
@@ -87,16 +94,31 @@ class MergeMethod(enum.Enum):
     def __str__(self) -> str:
         return self.value
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield str(cls.MERGE), "integrate the PR with a simple merge commit, using the PR description as message"
+        yield str(cls.REBASE_MERGE), "rebases the PR on top of the target branch the integrates with a merge commit, using the PR description as message"
+        yield str(cls.REBASE_FF), "rebases the PR on top of the target branch, then fast-forwards"
+        yield str(cls.SQUASH), "squashes the PR as a single commit on the target branch, using the PR description as message"
+
 
 class Retry:
     def __str__(self) -> str:
         return 'retry'
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield "retry", 're-tries staging a PR in the "error" state'
+
 
 class Check:
     def __str__(self) -> str:
         return 'check'
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield "check", "fetches or refreshes PR metadata, resets mergebot state"
+
 
 @dataclass
 class Override:
@@ -105,6 +127,10 @@ class Override:
     def __str__(self) -> str:
         return f"override={','.join(self.statuses)}"
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield "override=<...>", "marks overridable statuses as successful"
+
 
 @dataclass
 class Delegate:
@@ -115,6 +141,11 @@ class Delegate:
             return 'delegate+'
         return f"delegate={','.join(self.users)}"
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield "delegate+", "grants approval rights to the PR author"
+        yield "delegate=<...>", "grants approval rights on this PR to the specified github users"
+
 
 class Priority(enum.Enum):
     DEFAULT = enum.auto()
@@ -124,16 +155,30 @@ class Priority(enum.Enum):
     def __str__(self) -> str:
         return self.name.lower()
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield str(cls.DEFAULT), "stages the PR normally"
+        yield str(cls.PRIORITY), "tries to stage this PR first, then adds `default` PRs if the staging has room"
+        yield str(cls.ALONE), "stages this PR only with other PRs of the same priority"
+
 
 class CancelStaging:
     def __str__(self) -> str:
         return "cancel=staging"
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield "cancel=staging", "automatically cancels the current staging when this PR becomes ready"
+
 
 class SkipChecks:
     def __str__(self) -> str:
         return 'skipchecks'
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield "skipchecks", "bypasses both statuses and review"
+
 
 class FW(enum.Enum):
     DEFAULT = enum.auto()
@@ -144,6 +189,13 @@ class FW(enum.Enum):
     def __str__(self) -> str:
         return f'fw={self.name.lower()}'
 
+    @classmethod
+    def help(cls, is_reviewer: bool) -> Iterator[Tuple[str, str]]:
+        yield str(cls.NO), "does not forward-port this PR"
+        if is_reviewer:
+            yield str(cls.DEFAULT), "forward-ports this PR normally"
+            yield str(cls.SKIPCI), "does not wait for a forward-port's statuses to succeed before creating the next one"
+
 
 @dataclass
 class Limit:
@@ -154,11 +206,28 @@ class Limit:
             return 'ignore'
         return f'up to {self.branch}'
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield "up to <branch>", "only ports this PR forward to the specified branch (included)"
+
 
 class Close:
     def __str__(self) -> str:
         return 'close'
 
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield str(cls()), "closes this forward-port"
+
+
+class Help:
+    def __str__(self) -> str:
+        return 'help'
+
+    @classmethod
+    def help(cls, _: bool) -> Iterator[Tuple[str, str]]:
+        yield str(cls()), "displays this help"
+
 
 Command = Union[
     Approve,
@@ -167,6 +236,7 @@ Command = Union[
     Check,
     Delegate,
     FW,
+    Help,
     Limit,
     MergeMethod,
     Override,
@@ -309,3 +379,6 @@ class Parser:
 
     def parse_close(self) -> Close:
         return Close()
+
+    def parse_help(self) -> Help:
+        return Help()
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 9adb5ae0..a0a44ebc 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -642,6 +642,51 @@ class PullRequests(models.Model):
                 'message': message,
                 'close': close,
             })
+
+        is_admin, is_reviewer, is_author = self._pr_acl(author)
+        _source_admin, source_reviewer, source_author = self.source_id._pr_acl(author)
+        # nota: 15.0 `has_group` completely doesn't work if the recordset is empty
+        super_admin = is_admin and author.user_ids and author.user_ids.has_group('runbot_merge.group_admin')
+
+        help_list: list[type(commands.Command)] = list(filter(None, [
+            commands.Help,
+
+            (self.source_id and (source_author or source_reviewer) or is_reviewer) and not self.reviewed_by and commands.Approve,
+            is_author and self.reviewed_by and commands.Reject,
+            (is_author or source_author) and self.error and commands.Retry,
+
+            is_author and not self.source_id and commands.FW,
+            is_author and commands.Limit,
+            source_author and self.source_id and commands.Close,
+
+            is_reviewer and commands.MergeMethod,
+            is_reviewer and commands.Delegate,
+
+            is_admin and commands.Priority,
+            super_admin and commands.SkipChecks,
+            is_admin and commands.CancelStaging,
+
+            author.override_rights and commands.Override,
+            is_author and commands.Check,
+        ]))
+        def format_help(warn_ignore: bool, address: bool = True) -> str:
+            s = [
+                'Currently available commands{}:'.format(
+                    f" for @{login}" if address else ""
+                ),
+                '',
+                '|command||',
+                '|-|-|',
+            ]
+            for command_type in help_list:
+                for cmd, text in command_type.help(is_reviewer):
+                    s.append(f"|`{cmd}`|{text}|")
+
+            s.extend(['', 'Note: this help text is dynamic and will change with the state of the PR.'])
+            if warn_ignore:
+                s.extend(["", "Warning: in invoking help, every other command has been ignored."])
+            return "\n".join(s)
+
         try:
             cmds: List[commands.Command] = [
                 ps
@@ -656,11 +701,21 @@ class PullRequests(models.Model):
                 utils.shorten(comment['body'] or '', 50),
                 exc_info=True
             )
-            feedback(message=f"@{login} {e.args[0]}.\n\nFor your own safety I've ignored *everything in your entire comment*.")
+            feedback(message=f"""@{login} {e.args[0]}.
+
+For your own safety I've ignored *everything in your entire comment*.
+
+{format_help(False, address=False)}
+""")
             return 'error'
 
-        is_admin, is_reviewer, is_author = self._pr_acl(author)
-        _source_admin, source_reviewer, source_author = self.source_id._pr_acl(author)
+        if any(isinstance(cmd, commands.Help) for cmd in cmds):
+            self.env['runbot_merge.pull_requests.feedback'].create({
+                'repository': self.repository.id,
+                'pull_request': self.number,
+                'message': format_help(len(cmds) != 1),
+            })
+            return "help"
 
         if not (is_author or self.source_id or (any(isinstance(cmd, commands.Override) for cmd in cmds) and author.override_rights)):
             # no point even parsing commands
@@ -775,7 +830,7 @@ class PullRequests(models.Model):
                     delegates.write({'delegate_reviewer': [(4, self.id, 0)]})
                 case commands.Priority() if is_admin:
                     self.batch_id.priority = str(command)
-                case commands.SkipChecks() if is_admin:
+                case commands.SkipChecks() if super_admin:
                     self.batch_id.skipchecks = True
                     self.reviewed_by = author
                     if not (self.squash or self.merge_method):
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index d64596e6..9133e48b 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -2801,6 +2801,7 @@ class TestBatching(object):
         ]
         assert not pr22.staging_id
 
+    @pytest.mark.usefixtures("reviewer_admin")
     def test_batching_urgent(self, env, repo, config):
         with repo:
             m = repo.make_commit(None, 'initial', None, tree={'a': 'some content'})
@@ -2920,6 +2921,7 @@ class TestBatching(object):
         assert p_01.skipchecks == False
         assert p_01.cancel_staging == True
 
+    @pytest.mark.usefixtures("reviewer_admin")
     def test_batching_urgenter_than_split(self, env, repo, config):
         """ p=alone PRs should take priority over split stagings (processing
         of a staging having CI-failed and being split into sub-stagings)
@@ -2959,6 +2961,7 @@ class TestBatching(object):
         assert not p_1.staging_id
         assert to_pr(env, pr0).staging_id
 
+    @pytest.mark.usefixtures("reviewer_admin")
     def test_urgent_failed(self, env, repo, config):
         """ Ensure pr[p=0,state=failed] don't get picked up
         """
@@ -3306,6 +3309,7 @@ class TestReviewing:
         env.run_crons()
         assert to_pr(env, pr).state == 'approved'
 
+    @pytest.mark.usefixtures("reviewer_admin")
     def test_skipchecks(self, env, repo, users, config):
         """Skipcheck makes the PR immediately ready (if it's not in error or
         something)
@@ -3622,8 +3626,66 @@ class TestRecognizeCommands:
             (users['reviewer'], "hansen do the thing"),
             (users['reviewer'], "hansen @bobby-b r+ :+1:"),
             seen(env, pr, users),
-            (users['user'], "@{reviewer} unknown command 'do'.\n\nFor your own safety I've ignored *everything in your entire comment*.".format_map(users)),
-            (users['user'], "@{reviewer} unknown command '@bobby-b'.\n\nFor your own safety I've ignored *everything in your entire comment*.".format_map(users)),
+            (users['user'], """\
+@{reviewer} unknown command 'do'.
+
+For your own safety I've ignored *everything in your entire comment*.
+
+Currently available commands:
+
+|command||
+|-|-|
+|`help`|displays this help|
+|`r(eview)+`|approves the PR, if it's a forwardport also approves all non-detached parents|
+|`r(eview)=<number>`|only approves the specified parents|
+|`fw=no`|does not forward-port this PR|
+|`fw=default`|forward-ports this PR normally|
+|`fw=skipci`|does not wait for a forward-port's statuses to succeed before creating the next one|
+|`up to <branch>`|only ports this PR forward to the specified branch (included)|
+|`merge`|integrate the PR with a simple merge commit, using the PR description as message|
+|`rebase-merge`|rebases the PR on top of the target branch the integrates with a merge commit, using the PR description as message|
+|`rebase-ff`|rebases the PR on top of the target branch, then fast-forwards|
+|`squash`|squashes the PR as a single commit on the target branch, using the PR description as message|
+|`delegate+`|grants approval rights to the PR author|
+|`delegate=<...>`|grants approval rights on this PR to the specified github users|
+|`default`|stages the PR normally|
+|`priority`|tries to stage this PR first, then adds `default` PRs if the staging has room|
+|`alone`|stages this PR only with other PRs of the same priority|
+|`cancel=staging`|automatically cancels the current staging when this PR becomes ready|
+|`check`|fetches or refreshes PR metadata, resets mergebot state|
+
+Note: this help text is dynamic and will change with the state of the PR.
+""".format_map(users)),
+            (users['user'], """\
+@{reviewer} unknown command '@bobby-b'.
+
+For your own safety I've ignored *everything in your entire comment*.
+
+Currently available commands:
+
+|command||
+|-|-|
+|`help`|displays this help|
+|`r(eview)+`|approves the PR, if it's a forwardport also approves all non-detached parents|
+|`r(eview)=<number>`|only approves the specified parents|
+|`fw=no`|does not forward-port this PR|
+|`fw=default`|forward-ports this PR normally|
+|`fw=skipci`|does not wait for a forward-port's statuses to succeed before creating the next one|
+|`up to <branch>`|only ports this PR forward to the specified branch (included)|
+|`merge`|integrate the PR with a simple merge commit, using the PR description as message|
+|`rebase-merge`|rebases the PR on top of the target branch the integrates with a merge commit, using the PR description as message|
+|`rebase-ff`|rebases the PR on top of the target branch, then fast-forwards|
+|`squash`|squashes the PR as a single commit on the target branch, using the PR description as message|
+|`delegate+`|grants approval rights to the PR author|
+|`delegate=<...>`|grants approval rights on this PR to the specified github users|
+|`default`|stages the PR normally|
+|`priority`|tries to stage this PR first, then adds `default` PRs if the staging has room|
+|`alone`|stages this PR only with other PRs of the same priority|
+|`cancel=staging`|automatically cancels the current staging when this PR becomes ready|
+|`check`|fetches or refreshes PR metadata, resets mergebot state|
+
+Note: this help text is dynamic and will change with the state of the PR.
+""".format_map(users)),
         ]
 
 class TestRMinus:
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 07e42786..35a6bcbc 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -761,6 +761,7 @@ class TestMultiBatches:
         assert sp.mapped('batch_ids.prs') == \
             prs[2][0] | prs[2][1] | prs[3][0] | prs[3][1] | prs[4][0]
 
+@pytest.mark.usefixtures("reviewer_admin")
 def test_urgent(env, repo_a, repo_b, config):
     """ Either PR of a co-dependent pair being prioritised leads to the entire
     pair being prioritized
@@ -872,6 +873,7 @@ class TestBlocked:
         p = to_pr(env, pr)
         assert not p.blocked
 
+    @pytest.mark.usefixtures("reviewer_admin")
     def test_linked_unready(self, env, repo_a, repo_b, config):
         """ Create a PR A linked to a non-ready PR B,
         * A is blocked by default
diff --git a/runbot_merge/tests/test_oddities.py b/runbot_merge/tests/test_oddities.py
index 081bc3e3..1454165b 100644
--- a/runbot_merge/tests/test_oddities.py
+++ b/runbot_merge/tests/test_oddities.py
@@ -1,7 +1,7 @@
 import pytest
 import requests
 
-from utils import Commit, to_pr
+from utils import Commit, to_pr, seen
 
 
 def test_partner_merge(env):
@@ -253,8 +253,8 @@ def test_force_ready(env, repo, config):
     with repo:
         [m] = repo.make_commits(None, Commit('initial', tree={'m': 'm'}), ref="heads/master")
 
-        [c] = repo.make_commits(m, Commit('first', tree={'m': 'c1'}), ref="heads/other")
-        pr = repo.make_pr(title='title', body='body', target='master', head=c)
+        repo.make_commits(m, Commit('first', tree={'m': 'c1'}), ref="heads/other")
+        pr = repo.make_pr(target='master', head='other')
     env.run_crons()
 
     pr_id = to_pr(env, pr)
@@ -264,3 +264,102 @@ def test_force_ready(env, repo, config):
     assert pr_id.status == 'pending'
     reviewer = env['res.users'].browse([env._uid]).partner_id
     assert pr_id.reviewed_by == reviewer
+
+def test_help(env, repo, config, users, partners):
+    with repo:
+        [m] = repo.make_commits(None, Commit('initial', tree={'m': 'm'}), ref="heads/master")
+
+        repo.make_commits(m, Commit('first', tree={'m': 'c1'}), ref="heads/other")
+        pr = repo.make_pr(target='master', head='other')
+    env.run_crons()
+
+    for role in ['reviewer', 'self_reviewer', 'user', 'other']:
+        v = config[f'role_{role}']
+        with repo:
+            pr.post_comment("hansen help", v['token'])
+    with repo:
+        pr.post_comment("hansen r+ help", config['role_reviewer']['token'])
+
+    assert not partners['reviewer'].user_ids, "the reviewer should not be an internal user"
+
+    group_internal = env.ref("base.group_user")
+    group_admin = env.ref("runbot_merge.group_admin")
+    env['res.users'].create({
+        'partner_id': partners['reviewer'].id,
+        'login': 'reviewer',
+        'groups_id': [(4, group_internal.id, 0), (4, group_admin.id, 0)],
+    })
+
+    with repo:
+        pr.post_comment("hansen help", config['role_reviewer']['token'])
+    env.run_crons()
+
+    assert pr.comments == [
+        seen(env, pr, users),
+        (users['reviewer'], "hansen help"),
+        (users['self_reviewer'], "hansen help"),
+        (users['user'], "hansen help"),
+        (users['other'], "hansen help"),
+        (users['reviewer'], "hansen r+ help"),
+        (users['reviewer'], "hansen help"),
+        (users['user'], REVIEWER.format(user=users['reviewer'], skip="")),
+        (users['user'], RANDO.format(user=users['self_reviewer'])),
+        (users['user'], AUTHOR.format(user=users['user'])),
+        (users['user'], RANDO.format(user=users['other'])),
+        (users['user'],
+         REVIEWER.format(user=users['reviewer'], skip='')
+         + "\n\nWarning: in invoking help, every other command has been ignored."),
+        (users['user'], REVIEWER.format(
+            user=users['reviewer'],
+            skip='|`skipchecks`|bypasses both statuses and review|\n',
+        )),
+    ]
+
+REVIEWER = """\
+Currently available commands for @{user}:
+
+|command||
+|-|-|
+|`help`|displays this help|
+|`r(eview)+`|approves the PR, if it's a forwardport also approves all non-detached parents|
+|`r(eview)=<number>`|only approves the specified parents|
+|`fw=no`|does not forward-port this PR|
+|`fw=default`|forward-ports this PR normally|
+|`fw=skipci`|does not wait for a forward-port's statuses to succeed before creating the next one|
+|`up to <branch>`|only ports this PR forward to the specified branch (included)|
+|`merge`|integrate the PR with a simple merge commit, using the PR description as message|
+|`rebase-merge`|rebases the PR on top of the target branch the integrates with a merge commit, using the PR description as message|
+|`rebase-ff`|rebases the PR on top of the target branch, then fast-forwards|
+|`squash`|squashes the PR as a single commit on the target branch, using the PR description as message|
+|`delegate+`|grants approval rights to the PR author|
+|`delegate=<...>`|grants approval rights on this PR to the specified github users|
+|`default`|stages the PR normally|
+|`priority`|tries to stage this PR first, then adds `default` PRs if the staging has room|
+|`alone`|stages this PR only with other PRs of the same priority|
+{skip}\
+|`cancel=staging`|automatically cancels the current staging when this PR becomes ready|
+|`check`|fetches or refreshes PR metadata, resets mergebot state|
+
+Note: this help text is dynamic and will change with the state of the PR.\
+"""
+AUTHOR = """\
+Currently available commands for @{user}:
+
+|command||
+|-|-|
+|`help`|displays this help|
+|`fw=no`|does not forward-port this PR|
+|`up to <branch>`|only ports this PR forward to the specified branch (included)|
+|`check`|fetches or refreshes PR metadata, resets mergebot state|
+
+Note: this help text is dynamic and will change with the state of the PR.\
+"""
+RANDO = """\
+Currently available commands for @{user}:
+
+|command||
+|-|-|
+|`help`|displays this help|
+
+Note: this help text is dynamic and will change with the state of the PR.\
+"""

From 0a839a4857e3c0d6999321efbbb6ed7c6b53a170 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 25 Jun 2024 13:11:32 +0200
Subject: [PATCH 145/192] [FIX] forwardport: don't break forward porting on
 huge conflicts

On forward-porting, odoo/odoo#170183 generates a conflict on pretty
much every one of the 1111 files it touches, because they are
modify/delete conflicts that generates a conflict message over 200
bytes per file, which is over 200kB of output.

For this specific scenario, the commit message was being passed
through arguments to the `git` command, resulting in a command line
exceeding `MAX_ARG_STRLEN`[^1]. The obvious way to fix this is to pass
the commit message via stdin as is done literally in the line above
where we just copy a non-generated commit message.

However I don't think hundreds of kbytes worth of stdout[^2] is of any
use, so shorten that a bit, and stderr while at it.

Don't touch the commit message size for now, possibly forever, but
note that some log diving reveals a commit with a legit 18kB message
(odoo/odoo@42a3b704f7c7889a74338253138d0201d3b6e7a3) so if we want to
restrict that the limit should be at least 32k, and possibly 64. But
it might be a good idea to make that limit part of the ready / merge
checks too, rather than cut things off or trigger errors during
staging.

Fixes #900

[^1]: Most resources on "Argument list too long" reference `ARG_MAX`,
    but on both my machine and the server it is 2097152 (25% of the
    default stack), which is ~10x larger than the commit message we
    tried to generate. The actual limit is `MAX_ARG_STRLEN` which
    can't be queried directly but is essentially hard-coded to
    PAGE_SIZE * 32 = 128kiB, which tracks.

[^2]: Somewhat unexpectedly, that's where `git-cherry-pick` sends the
    conflict info.
---
 forwardport/models/project.py       | 29 +++++-----
 forwardport/tests/test_conflicts.py | 89 +++++++++++++++++++++++++++++
 runbot_merge/utils.py               |  5 +-
 3 files changed, 106 insertions(+), 17 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index b0d743e8..c4024c23 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -32,11 +32,10 @@ from odoo.osv import expression
 from odoo.tools.misc import topological_sort, groupby
 from odoo.tools.appdirs import user_cache_dir
 from odoo.addons.base.models.res_partner import Partner
-from odoo.addons.runbot_merge import git
+from odoo.addons.runbot_merge import git, utils
 from odoo.addons.runbot_merge.models.pull_requests import Branch
 from odoo.addons.runbot_merge.models.stagings_create import Message
 
-
 DEFAULT_DELTA = dateutil.relativedelta.relativedelta(days=3)
 
 _logger = logging.getLogger('odoo.addons.forwardport')
@@ -378,7 +377,7 @@ class PullRequests(models.Model):
                 authors.add(to_tuple(c['author']))
                 committers.add(to_tuple(c['committer']))
             fp_authorship = (project_id.fp_github_name, '', '')
-            author = fp_authorship if len(authors) != 1\
+            author = fp_authorship if len(authors) != 1 \
                 else authors.pop() + (head_commit['author']['date'],)
             committer = fp_authorship if len(committers) != 1 \
                 else committers.pop() + (head_commit['committer']['date'],)
@@ -399,31 +398,33 @@ class PullRequests(models.Model):
                 'merge.renamelimit=0',
                 'merge.renames=copies',
                 'merge.conflictstyle=zdiff3'
-            )\
-                .with_config(check=False)\
+            ) \
+                .with_config(check=False) \
                 .cherry_pick(squashed, no_commit=True, strategy="ort")
             status = conf.stdout().status(short=True, untracked_files='no').stdout.decode()
             if err.strip():
                 err = err.rstrip() + '\n----------\nstatus:\n' + status
             else:
                 err = 'status:\n' + status
+
             # if there was a single commit, reuse its message when committing
             # the conflict
-            # TODO: still add conflict information to this?
             if len(commits) == 1:
                 msg = root._make_fp_message(commits[0])
-                conf.with_config(input=str(msg).encode()) \
-                    .commit(all=True, allow_empty=True, file='-')
             else:
-                conf.commit(
-                    all=True, allow_empty=True,
-                    message="""Cherry pick of %s failed
+                out = utils.shorten(out, 8*1024, '[...]')
+                err = utils.shorten(err, 8*1024, '[...]')
+                msg = f"""Cherry pick of {h} failed
 
 stdout:
-%s
+{out}
 stderr:
-%s
-""" % (h, out, err))
+{err}
+"""
+
+            conf.with_config(input=str(msg).encode()) \
+                .commit(all=True, allow_empty=True, file='-')
+
             return (h, out, err, [c['sha'] for c in commits]), working_copy
 
     def _cherry_pick(self, working_copy):
diff --git a/forwardport/tests/test_conflicts.py b/forwardport/tests/test_conflicts.py
index 4172f988..030e534b 100644
--- a/forwardport/tests/test_conflicts.py
+++ b/forwardport/tests/test_conflicts.py
@@ -1,3 +1,4 @@
+import random
 import re
 import time
 from operator import itemgetter
@@ -177,6 +178,94 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
         'i': 'a',
     }
 
+def test_massive_conflict(env, config, make_repo):
+    """If the conflict is large enough, the commit message may exceed ARG_MAX
+     and trigger E2BIG.
+    """
+    # CONFLICT (modify/delete): <file> deleted in <commit> (<title>) and modified in HEAD.  Version HEAD of <file> left in tree.
+    #
+    # 107 + 2 * len(filename) + len(title) per conflicting file.
+    # - filename: random.randbytes(10).hex() -> 20
+    # - title: random.randbytes(20).hex() -> 40
+    # -> 701 (!) files
+
+    files = []
+    while len(files) < 1500:
+        files.append(random.randbytes(10).hex())
+
+    # region setup
+    project = env['runbot_merge.project'].create({
+        'name': "thing",
+        'github_token': config['github']['token'],
+        'github_prefix': 'hansen',
+        'fp_github_token': config['github']['token'],
+        'fp_github_name': 'herbert',
+        'branch_ids': [
+            (0, 0, {'name': 'a', 'sequence': 100}),
+            (0, 0, {'name': 'b', 'sequence': 80}),
+        ],
+    })
+
+    repo = make_repo("repo")
+    env['runbot_merge.events_sources'].create({'repository': repo.name})
+
+    repo_id = env['runbot_merge.repository'].create({
+        'project_id': project.id,
+        'name': repo.name,
+        'required_statuses': "default",
+        'fp_remote_target': repo.name,
+        'group_id': False,
+    })
+    env['res.partner'].search([
+        ('github_login', '=', config['role_reviewer']['user'])
+    ]).write({
+        'review_rights': [(0, 0, {'repository_id': repo_id.id, 'review': True})]
+    })
+
+    with repo:
+        # create branch with a ton of empty files
+        repo.make_commits(
+            None,
+            Commit(
+                random.randbytes(20).hex(),
+                tree=dict.fromkeys(files, "xoxo"),
+            ),
+            ref='heads/a',
+        )
+
+        # removes all those files in the next branch
+        repo.make_commits(
+            'a',
+            Commit(
+                random.randbytes(20).hex(),
+                tree=dict.fromkeys(files, "content!"),
+            ),
+            ref='heads/b',
+        )
+    # endregion setup
+
+    with repo:
+        # update all the files
+        repo.make_commits(
+            'a',
+            Commit(random.randbytes(20).hex(), tree={'a': '1'}),
+            Commit(random.randbytes(20).hex(), tree={'x': '1'}, reset=True),
+            ref='heads/change',
+        )
+        pr = repo.make_pr(target='a', head='change')
+        repo.post_status('refs/heads/change', 'success')
+        pr.post_comment('hansen rebase-ff r+', config['role_reviewer']['token'])
+    env.run_crons()
+
+    with repo:
+        repo.post_status('staging.a', 'success')
+    env.run_crons()
+
+    # we don't actually need more, the bug crashes the forward port entirely so
+    # the PR is never even created
+    _pra_id, _prb_id = env['runbot_merge.pull_requests'].search([], order='number')
+
+
 def test_conflict_deleted(env, config, make_repo):
     prod, other = make_basic(env, config, make_repo)
     # remove f from b
diff --git a/runbot_merge/utils.py b/runbot_merge/utils.py
index bd35d3db..0ba2cc49 100644
--- a/runbot_merge/utils.py
+++ b/runbot_merge/utils.py
@@ -3,7 +3,7 @@ import itertools
 import time
 
 
-def shorten(text_ish, length):
+def shorten(text_ish, length, cont='...'):
     """ If necessary, cuts-off the text or bytes input and appends ellipsis to
     signal the cutoff, such that the result is below the provided length
     (according to whatever "len" means on the text-ish so bytes or codepoints
@@ -12,11 +12,10 @@ def shorten(text_ish, length):
     if len(text_ish or ()) <= length:
         return text_ish
 
-    cont = '...'
     if isinstance(text_ish, bytes):
         cont = cont.encode('ascii') # whatever
     # add enough room for the ellipsis
-    return text_ish[:length-3] + cont
+    return text_ish[:length-len(cont)] + cont
 
 BACKOFF_DELAYS = (0.1, 0.2, 0.4, 0.8, 1.6)
 def backoff(func=None, *, delays=BACKOFF_DELAYS, exc=Exception):

From 286c1fdaee12bce073c28a48e4e415e843de545f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 25 Jun 2024 13:27:16 +0200
Subject: [PATCH 146/192] [FIX] runbot_merge: allow source author to r- forward
 ports

Noticed that while writing up the docs on the wiki, seems like an
unnecessary restriction, and an inconvenient one to boot: the author
could r+, then realize they forgot to do an update they needed to do
on the fw, so they should be able to cancel the staging without
needing a reviewer.
---
 runbot_merge/models/pull_requests.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index a0a44ebc..76bc6a9f 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -652,7 +652,7 @@ class PullRequests(models.Model):
             commands.Help,
 
             (self.source_id and (source_author or source_reviewer) or is_reviewer) and not self.reviewed_by and commands.Approve,
-            is_author and self.reviewed_by and commands.Reject,
+            (is_author or source_author) and self.reviewed_by and commands.Reject,
             (is_author or source_author) and self.error and commands.Retry,
 
             is_author and not self.source_id and commands.FW,
@@ -783,7 +783,7 @@ For your own safety I've ignored *everything in your entire comment*.
                         msg = f"tried to approve PRs {command.ids} but the current PR is {self.number}"
                     else:
                         msg = self._approve(author, login)
-                case commands.Reject() if is_author:
+                case commands.Reject() if is_author or source_author:
                     if self.batch_id.skipchecks or self.reviewed_by:
                         if self.error:
                             self.error = False

From 6ada35a20033e6701188cb7d32cc2facbb95093d Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 25 Jun 2024 15:05:32 +0200
Subject: [PATCH 147/192] [CHG] runbot_merge: ignore long comments

Comments which are too long cause `logging` itself to crash, which
kinda sucks. And long comments seem very unlikely to have anything for
the mergebot to do besides.

So just ignore them at intake. Limit is set to 5000 because there
needs to be a limit somewhere and that's about the extent of it.
---
 runbot_merge/controllers/__init__.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 278b6609..1fb16fa1 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -386,6 +386,10 @@ def handle_comment(env, event):
     issue = event['issue']['number']
     author = event['comment']['user']['login']
     comment = event['comment']['body']
+    if len(comment) > 5000:
+        _logger.warning('comment(%s): %s %s#%s => ignored (%d characters)', event['comment']['html_url'], author, repo, issue, len(comment))
+        return "ignored: too big"
+
     _logger.info('comment[%s]: %s %s#%s %r', event['action'], author, repo, issue, comment)
     if event['action'] != 'created':
         return "Ignored: action (%r) is not 'created'" % event['action']
@@ -397,6 +401,9 @@ def handle_review(env, event):
     pr = event['pull_request']['number']
     author = event['review']['user']['login']
     comment = event['review']['body'] or ''
+    if len(comment) > 5000:
+        _logger.warning('comment(%s): %s %s#%s => ignored (%d characters)', event['review']['html_url'], author, repo, pr, len(comment))
+        return "ignored: too big"
 
     _logger.info('review[%s]: %s %s#%s %r', event['action'], author, repo, pr, comment)
     if event['action'] != 'submitted':

From 3bc5b4e3e4a413eef576494f8d15fa1fca5e1533 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 25 Jun 2024 15:06:41 +0200
Subject: [PATCH 148/192] [CHG] runbot_merge: log ping instead of printing it

That's an old and completely useless leftover, but I never got around
to swapping it. It could be removed entirely or moved to debug as
well...
---
 runbot_merge/controllers/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 1fb16fa1..b8005979 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -414,7 +414,7 @@ def handle_review(env, event):
         target=event['pull_request']['base']['ref'])
 
 def handle_ping(env, event):
-    print("Got ping! {}".format(event['zen']))
+    _logger.info("Got ping! %s", event['zen'])
     return "pong"
 
 EVENTS = {

From 1c76a675c28bb2b120927682afca29e5334d472f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 26 Jun 2024 14:30:31 +0200
Subject: [PATCH 149/192] [IMP] runbot_merge: cancel splits on cancel=staging

If a PR is cancel=staging, even if it's not the
urgentest (priority=alone) odds are good it's being staged to fix the
split. And even if it's not, it probably can't hurt.

So cancel splits in order to stage it. This may be slightly harmful if
the split is legit and has nothing to do with the PR being
prioritised, but that seems like the less likely scenario. And having
to update staging priorities on the fly seems like a bad idea. Though
obviously it might do nothing if the PRs are in "default" priority.#

Also simplify the unstage trigger from the PRs becoming ready:

- the user is useless as it's always the system user
- the batch id is not really helpful
---
 runbot_merge/models/batch.py         | 6 +++---
 runbot_merge/models/pull_requests.py | 2 ++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index fbca57fc..d35f743d 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -227,10 +227,10 @@ class Batch(models.Model):
                 batch.blocked = f"Multiple target branches: {', '.join(targets.mapped('name'))!r}"
             else:
                 if batch.blocked and batch.cancel_staging:
+                    if splits := batch.target.split_ids:
+                        splits.unlink()
                     batch.target.active_staging_id.cancel(
-                        'unstaged by %s on %s (%s)',
-                        self.env.user.login,
-                        batch,
+                        'unstaged by %s becoming ready',
                         ', '.join(batch.prs.mapped('display_name')),
                     )
                 batch.blocked = False
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 76bc6a9f..927536af 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -842,6 +842,8 @@ For your own safety I've ignored *everything in your entire comment*.
                 case commands.CancelStaging() if is_admin:
                     self.batch_id.cancel_staging = True
                     if not self.batch_id.blocked:
+                        if splits := self.target.split_ids:
+                            splits.unlink()
                         self.target.active_staging_id.cancel(
                             "Unstaged by %s on %s",
                             author.github_login, self.display_name,

From de32824a6279fc742a00c68dec64b002a98e6348 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 26 Jun 2024 14:34:34 +0200
Subject: [PATCH 150/192] [IMP] *: move the `page` helper fixture to the shared
 conftest

Use it in `test_limit` instead of direct `requests` calls.
---
 conftest.py                     |  9 +++++++++
 forwardport/tests/test_limit.py |  9 +++------
 runbot_merge/tests/conftest.py  | 10 ----------
 3 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/conftest.py b/conftest.py
index 9572cea3..99b824e5 100644
--- a/conftest.py
+++ b/conftest.py
@@ -385,6 +385,15 @@ def port():
         s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         return s.getsockname()[1]
 
+@pytest.fixture
+def page(port):
+    with requests.Session() as s:
+        def get(url):
+            r = s.get('http://localhost:{}{}'.format(port, url))
+            r.raise_for_status()
+            return r.content
+        yield get
+
 @pytest.fixture(scope='session')
 def dummy_addons_path():
     with tempfile.TemporaryDirectory() as dummy_addons_path:
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index 3e7bb9ed..1dd76980 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -1,5 +1,4 @@
 import pytest
-import requests
 
 from utils import seen, Commit, make_basic, to_pr
 
@@ -9,7 +8,7 @@ from utils import seen, Commit, make_basic, to_pr
     pytest.param('b', 'b', 0, id='current'),
     pytest.param('b', 'a', 0, id='earlier'),
 ])
-def test_configure_fp_limit(env, config, make_repo, source, limit, count, port):
+def test_configure_fp_limit(env, config, make_repo, source, limit, count, page):
     prod, other = make_basic(env, config, make_repo, statuses="default")
     with prod:
         [c] = prod.make_commits(
@@ -35,15 +34,13 @@ def test_configure_fp_limit(env, config, make_repo, source, limit, count, port):
     assert not descendants.limit_id, "descendant should not inherit the limit explicitly"
 
     # check that the basic thingie works
-    r = requests.get(f'http://localhost:{port}/{prod.name}/pull/{pr.number}.png')
-    assert r.ok, r.text
+    page(f'/{prod.name}/pull/{pr.number}.png')
 
     if descendants:
         c = env['runbot_merge.branch'].search([('name', '=', 'c')])
         descendants.limit_id = c.id
 
-        r = requests.get(f'http://localhost:{port}/{prod.name}/pull/{pr.number}.png')
-        assert r.ok
+        page(f'/{prod.name}/pull/{pr.number}.png')
 
 def test_ignore(env, config, make_repo, users):
     """ Provide an "ignore" command which is equivalent to setting the limit
diff --git a/runbot_merge/tests/conftest.py b/runbot_merge/tests/conftest.py
index f75447ca..b663a688 100644
--- a/runbot_merge/tests/conftest.py
+++ b/runbot_merge/tests/conftest.py
@@ -1,19 +1,9 @@
 import pytest
-import requests
 
 @pytest.fixture()
 def module():
     return 'runbot_merge'
 
-@pytest.fixture
-def page(port):
-    s = requests.Session()
-    def get(url):
-        r = s.get('http://localhost:{}{}'.format(port, url))
-        r.raise_for_status()
-        return r.content
-    return get
-
 @pytest.fixture
 def default_crons():
     return [

From eb23a8c08376e143a38da8305f31fd69d7b22607 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 26 Jun 2024 15:30:59 +0200
Subject: [PATCH 151/192] [REM] forwardport: useless test

`test_maintain_batch_history` was built for the original design where
PRs were removed from batches on being closed.

This decision was reverted in bbce5f8f46f270099e54b4a39c4e548798cabfd0
as it proved an inferior and inconvenient design even in the face of
some of the edge cases, however I clearly forgot about this test.
---
 forwardport/tests/test_weird.py | 55 ---------------------------------
 1 file changed, 55 deletions(-)

diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index d6cd732d..0daecba7 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -1112,61 +1112,6 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 """.format_map(users))
     ]
 
-def test_maintain_batch_history(env, config, make_repo, users):
-    """Batches which are part of a forward port sequence should not be deleted
-    even if all their PRs are closed.
-
-    Sadly in that case it's a bit difficult to maintain the integrity of the
-    batch as each PR being closed (until the last one?) will be removed from
-    the batch.
-    """
-    repo, fork = make_basic(env, config, make_repo, statuses="default")
-
-    with repo, fork:
-        fork.make_commits("a", Commit("x", tree={"x": "1"}), ref="heads/x")
-        pr1_a = repo.make_pr(title="X", target="a", head=f"{fork.owner}:x")
-        pr1_a.post_comment("hansen r+", config['role_reviewer']['token'])
-        repo.post_status(pr1_a.head, "success")
-    env.run_crons()
-
-    pr1_a_id = to_pr(env, pr1_a)
-    with repo:
-        repo.post_status('staging.a', 'success')
-    env.run_crons()
-
-    pr1_b_id = env['runbot_merge.pull_requests'].search([('parent_id', '=', pr1_a_id.id)])
-    with repo:
-        repo.post_status(pr1_b_id.head, 'success')
-    env.run_crons()
-
-    pr1_c_id = env['runbot_merge.pull_requests'].search([('parent_id', '=', pr1_b_id.id)])
-
-    # region check that all the batches are set up correctly
-    assert pr1_a_id.batch_id
-    assert pr1_b_id.batch_id
-    assert pr1_c_id.batch_id
-    assert pr1_c_id.batch_id.parent_id == pr1_b_id.batch_id
-    assert pr1_b_id.batch_id.parent_id == pr1_a_id.batch_id
-    b_batch = pr1_b_id.batch_id
-    assert b_batch
-    # endregion
-
-    pr1_b = repo.get_pr(pr1_b_id.number)
-    with repo:
-        pr1_b.close()
-    env.run_crons()
-    assert pr1_b_id.state == 'closed'
-
-    # region check that all the batches are *still* set up correctly
-    assert b_batch.exists()
-    assert pr1_a_id.batch_id == b_batch.parent_id
-    assert pr1_b_id.batch_id == b_batch
-    assert pr1_c_id.batch_id.parent_id == b_batch
-
-    assert pr1_b_id in b_batch.all_prs, "the PR is still in the batch"
-    assert pr1_b_id not in b_batch.prs, "the PR is not in the open/active batch PRs"
-    # endregion
-
 FMT = '%Y-%m-%d %H:%M:%S'
 FAKE_PREV_WEEK = (datetime.now() + timedelta(days=1)).strftime(FMT)
 def test_reminder_detached(env, config, make_repo, users):

From 318e55337cb128653ac445efcd9f1b0a598d1651 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 28 Jun 2024 08:18:34 +0200
Subject: [PATCH 152/192] [FIX] forwardport: count next to users should be the
 fwport

Previously it would count the number of source PRs with outstanding
forward ports, which is not the count from the home page so that was
confusing.

Also add counts next to the groups, so teams can be identified at a
glance.

And finally outline the current user in the list, so they can find
themselves faster when they're not one of the top entries.
---
 forwardport/controllers.py | 12 +++++++++---
 forwardport/data/views.xml | 13 ++++++++++---
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/forwardport/controllers.py b/forwardport/controllers.py
index 620eab96..7d7515d1 100644
--- a/forwardport/controllers.py
+++ b/forwardport/controllers.py
@@ -66,18 +66,23 @@ class Dashboard(MergebotDashboard):
             ('state', 'in', ['opened', 'validated', 'approved', 'ready', 'error']),
             ('source_id', 'in', PullRequests._search(source_filter)),
         ])
+
+        outstanding_per_group = collections.Counter()
         outstanding_per_author = collections.Counter()
         outstanding_per_reviewer = collections.Counter()
         outstandings = []
         for source in outstanding.mapped('source_id').sorted('merge_date'):
+            prs = source.forwardport_ids.filtered(lambda p: p.state not in ['merged', 'closed'])
             outstandings.append({
                 'source': source,
-                'prs': source.forwardport_ids.filtered(lambda p: p.state not in ['merged', 'closed']),
+                'prs': prs,
             })
             if authors:
-                outstanding_per_author[source.author] += 1
+                outstanding_per_author[source.author] += len(prs)
+                outstanding_per_group[source.author.commercial_partner_id] += len(prs)
             if reviewers and source:
-                outstanding_per_reviewer[source.reviewed_by] += 1
+                outstanding_per_reviewer[source.reviewed_by] += len(prs)
+                outstanding_per_group[source.reviewed_by.commercial_partner_id] += len(prs)
 
         culprits = Partners.browse(p.id for p, _ in (outstanding_per_reviewer + outstanding_per_author).most_common())
         return request.render('forwardport.outstanding', {
@@ -89,6 +94,7 @@ class Dashboard(MergebotDashboard):
             'current_group': group,
             'outstanding_per_author': outstanding_per_author,
             'outstanding_per_reviewer': outstanding_per_reviewer,
+            'outstanding_per_group': outstanding_per_group,
             'outstanding': outstandings,
             'link': link,
         })
diff --git a/forwardport/data/views.xml b/forwardport/data/views.xml
index be8480d9..f9d7f378 100644
--- a/forwardport/data/views.xml
+++ b/forwardport/data/views.xml
@@ -36,11 +36,17 @@
                     <t t-foreach="groups" t-as="group">
                         <span>
                             <t t-if="group == current_group">
-                                <span class="bg-primary" t-out="group.display_name"/>
+                                <span class="bg-primary">
+                                    <t t-out="group.display_name"/>
+                                    (<t t-out="outstanding_per_group[group]"/>)
+                                </span>
                                 <a t-att-href="link()" class="btn fa fa-times p-0"/>
                             </t>
                             <t t-else="">
-                                <a t-att-href="link(group=group.id, partner=0)" t-out="group.display_name"/>
+                                <a t-att-href="link(group=group.id, partner=0)">
+                                    <t t-out="group.display_name"/>
+                                    (<t t-out="outstanding_per_group[group]"/>)
+                                </a>
                             </t>
                         </span>
                     </t>
@@ -51,7 +57,8 @@
                         <t t-set="created" t-value="outstanding_per_author[culprit]"/>
                         <a t-att-href="link(partner=culprit.id)"
                            t-attf-title="approved {{approved}}, created {{created}}"
-                        ><t t-out="culprit.display_name"/>:
+                           t-att-class="'bg-primary' if culprit == env.user.partner_id else None"
+                        ><t t-out="culprit.name"/>:
                             <t t-if="approved" t-out="approved"/>
                             <t t-if="approved and created"> + </t>
                             <t t-if="created" t-out="created"/>

From 0206d5f977bf325b826c54cf7683b829bf52faab Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 28 Jun 2024 12:22:23 +0200
Subject: [PATCH 153/192] [FIX] runbot_merge: followup detached PRs when
 disabling branches

Although the handling of forward ports on disabled branch was improved
in 94fe0329b41cb59022ef68430399abd3cccad371 in order to avoid losing
or needing to manually port such, because it goes through
`_schedule_fw_followup` some of the tests *that* performs were missed,
most notably that it only ports batches when no PRs are detached.

This is an issue if we need to force the port because of a branch
being deactivated: the forward-port could have stopped there due to a
conflict, in which case it's always going to be detached.

Thus the `force_fw` flag should also override the parenting state
check.

Also while at it make `force_fw` a regular flag, I don't understand
why I made it into a context value in the first place, it's only
passed from one location and that's directy calling the one function
which uses it...

Fixes #897
---
 forwardport/models/project.py   | 11 +++++------
 forwardport/tests/test_weird.py | 19 +++++++++++++++----
 runbot_merge/models/batch.py    | 12 +++++++++---
 3 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index c4024c23..2b58bf86 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -83,12 +83,11 @@ class Project(models.Model):
                     # and has not already been forward ported
                     and Batch.search_count([('parent_id', '=', b.id)]) == 0
                 )
-                ported |= extant.prs.filtered(lambda p: p._find_next_target())
-                # enqueue a forward port as if the now deactivated branch had
-                # been skipped over (which is the normal fw behaviour)
-                for b in extant.with_context(force_fw=True):
-                    # otherwise enqueue a followup
-                    b._schedule_fp_followup()
+
+                # PRs may have different limits in the same batch so only notify
+                # those which actually needed porting
+                ported |= extant._schedule_fp_followup(force_fw=True)\
+                    .prs.filtered(lambda p: p._find_next_target())
 
         if not ported:
             return
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index 0daecba7..e2c573b6 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -957,7 +957,7 @@ def test_disable_branch_with_batches(env, config, make_repo, users):
     })
     # endregion
 
-    # region set up forward ported batch
+    # region set up forward ported batches
     with repo, fork, repo2, fork2:
         fork.make_commits("a", Commit("x", tree={"x": "1"}), ref="heads/x")
         pr1_a = repo.make_pr(title="X", target="a", head=f"{fork.owner}:x")
@@ -968,6 +968,11 @@ def test_disable_branch_with_batches(env, config, make_repo, users):
         pr2_a = repo2.make_pr(title="X", target="a", head=f"{fork2.owner}:x")
         pr2_a.post_comment("hansen r+", config['role_reviewer']['token'])
         repo2.post_status(pr2_a.head, "success")
+
+        fork.make_commits("a", Commit("y", tree={"y": "1"}), ref="heads/y")
+        pr3_a = repo.make_pr(title="Y", target="a", head=f"{fork.owner}:y")
+        pr3_a.post_comment("hansen r+", config['role_reviewer']['token'])
+        repo.post_status(pr3_a.head, 'success')
     # remove just pr2 from the forward ports (maybe?)
     pr2_a_id = to_pr(env, pr2_a)
     pr2_a_id.limit_id = branch_b.id
@@ -975,7 +980,6 @@ def test_disable_branch_with_batches(env, config, make_repo, users):
     assert pr2_a_id.limit_id == branch_b
     # endregion
 
-
     with repo, repo2:
         repo.post_status('staging.a', 'success')
         repo2.post_status('staging.a', 'success')
@@ -984,10 +988,15 @@ def test_disable_branch_with_batches(env, config, make_repo, users):
     PullRequests = env['runbot_merge.pull_requests']
     pr1_b_id = PullRequests.search([('parent_id', '=', to_pr(env, pr1_a).id)])
     pr2_b_id = PullRequests.search([('parent_id', '=', pr2_a_id.id)])
+    pr3_b_id = PullRequests.search([('parent_id', '=', to_pr(env, pr3_a).id)])
     assert pr1_b_id.parent_id
     assert pr1_b_id.state == 'opened'
     assert pr2_b_id.parent_id
     assert pr2_b_id.state == 'opened'
+    assert pr3_b_id.parent_id
+    assert pr3_b_id.state == 'opened'
+    # detach pr3 (?)
+    pr3_b_id.write({'parent_id': False, 'detach_reason': 'because'})
 
     b_id = proj.branch_ids.filtered(lambda b: b.name == 'b')
     proj.write({
@@ -995,8 +1004,10 @@ def test_disable_branch_with_batches(env, config, make_repo, users):
     })
     env.run_crons()
     assert not b_id.active
-    assert PullRequests.search_count([]) == 5, "should have ported pr1 but not pr2"
-    assert PullRequests.search([], order="number DESC", limit=1).parent_id == pr1_b_id
+    # pr1_a, pr1_b, pr1_c, pr2_a, pr2_b, pr3_a, pr3_b, pr3_c
+    assert PullRequests.search_count([]) == 8, "should have ported pr1 and pr3 but not pr2"
+    assert PullRequests.search_count([('parent_id', '=', pr1_b_id.id)])
+    assert PullRequests.search_count([('parent_id', '=', pr3_b_id.id)])
 
     assert repo.get_pr(pr1_b_id.number).comments == [
         seen(env, repo.get_pr(pr1_b_id.number), users),
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index d35f743d..0f0c82cf 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -415,7 +415,7 @@ class Batch(models.Model):
         new_batch._schedule_fp_followup()
         return new_batch
 
-    def _schedule_fp_followup(self):
+    def _schedule_fp_followup(self, *, force_fw=False):
         _logger = logging.getLogger(__name__).getChild('forwardport.next')
         # if the PR has a parent and is CI-validated, enqueue the next PR
         scheduled = self.browse(())
@@ -424,10 +424,16 @@ class Batch(models.Model):
             _logger.info('Checking if forward-port %s (%s)', batch, prs)
             # in cas of conflict or update individual PRs will "lose" their
             # parent, which should prevent forward porting
-            if not (batch.parent_id and all(p.parent_id for p in batch.prs)):
+            #
+            # even if we force_fw, a *followup* should still only be for forward
+            # ports so check that the batch has a parent (which should be the
+            # same thing as all the PRs having a source, kinda, but cheaper,
+            # it's not entirely true as technically the user could have added a
+            # PR to the forward ported batch
+            if not (batch.parent_id and force_fw or all(p.parent_id for p in batch.prs)):
                 _logger.info('-> no parent %s (%s)', batch, prs)
                 continue
-            if not self.env.context.get('force_fw') and batch.source.fw_policy != 'skipci' \
+            if not force_fw and batch.source.fw_policy != 'skipci' \
                     and (invalid := batch.prs.filtered(lambda p: p.state not in ['validated', 'ready'])):
                 _logger.info(
                     '-> wrong state %s (%s)',

From 94cf3e96478004a4cb51e153a2f5b2e4d46c4839 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 28 Jun 2024 16:03:03 +0200
Subject: [PATCH 154/192] [IMP] *: convert fw=no to a genuine forward-porting
 policy

After seeing it be used, I foresee confusion around the current
behaviour (where it sets the limit), as one would expect the `fw=`
flags to affect one another when it looks like that would make sense
e.g. no/default/skipci/skipmerge all specify how to forward port, so
`fw=default` not doing anything after you've said `fw=no` (possibly by
mistake) would be fucking weird.

Also since the author can set limits, allow them to reset the fw
policy to default (keep skipci for reviewers), and for @d-fence add a
`fw=disabled` alias.

Fixes #902
---
 forwardport/models/project.py        |  3 +++
 forwardport/tests/test_limit.py      |  4 +--
 runbot_merge/models/batch.py         |  5 ++--
 runbot_merge/models/commands.py      |  2 ++
 runbot_merge/models/pull_requests.py | 40 +++++++++-------------------
 5 files changed, 23 insertions(+), 31 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 2b58bf86..5c664c25 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -550,6 +550,9 @@ class Stagings(models.Model):
         if vals.get('active') is False and self.state == 'success':
             # check all batches to see if they should be forward ported
             for b in self.with_context(active_test=False).batch_ids:
+                if b.fw_policy == 'no':
+                    continue
+
                 # if all PRs of a batch have parents they're part of an FP
                 # sequence and thus handled separately, otherwise they're
                 # considered regular merges
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index 1dd76980..1e8fcf7b 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -72,8 +72,8 @@ def test_ignore(env, config, make_repo, users):
         (users['reviewer'], "hansen ignore"),
         (users['reviewer'], "hansen r+ fw=no"),
         (users['user'], "'ignore' is deprecated, use 'fw=no' to disable forward porting."),
-        (users['user'], "Forward-port disabled."),
-        (users['user'], "Forward-port disabled."),
+        (users['user'], "Forward-port disabled (via limit)."),
+        (users['user'], "Disabled forward-porting."),
     ]
 
 @pytest.mark.expect_log_errors(reason="one of the limit-setting does not provide a branch name")
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index 0f0c82cf..c85eb4f7 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -68,9 +68,10 @@ class Batch(models.Model):
     active = fields.Boolean(compute='_compute_active', store=True, help="closed batches (batches containing only closed PRs)")
 
     fw_policy = fields.Selection([
+        ('no', "Do not port forward"),
         ('default', "Default"),
         ('skipci', "Skip CI"),
-    ], required=True, default="default", string="Forward Port Policy")
+    ], required=True, default="default", string="Forward Port Policy", tracking=True)
 
     merge_date = fields.Datetime(tracking=True)
     # having skipchecks skip both validation *and approval* makes sense because
@@ -89,7 +90,7 @@ class Batch(models.Model):
         ('default', "Default"),
         ('priority', "Priority"),
         ('alone', "Alone"),
-    ], default='default', group_operator=None, required=True,
+    ], default='default', group_operator=None, required=True, tracking=True,
         column_type=enum(_name, 'priority'),
     )
 
diff --git a/runbot_merge/models/commands.py b/runbot_merge/models/commands.py
index 684209e3..675d71eb 100644
--- a/runbot_merge/models/commands.py
+++ b/runbot_merge/models/commands.py
@@ -363,6 +363,8 @@ class Parser:
         self.assert_next('=')
         f = next(self.it, "")
         try:
+            if f in ('disable', 'disabled'):
+                return FW.NO
             return FW[f.upper()]
         except KeyError:
             raise CommandError(f"unknown fw configuration {f or None!r}") from None
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 927536af..75c76880 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -873,35 +873,21 @@ For your own safety I've ignored *everything in your entire comment*.
                 case commands.Close() if source_author:
                     feedback(close=True)
                 case commands.FW():
-                    if command == commands.FW.NO:
-                        if is_author:
-                            for p in self.batch_id.prs:
-                                ping, m = p._maybe_update_limit(self.target.name)
+                    match command:
+                        case commands.FW.NO if is_author or source_author:
+                            message = "Disabled forward-porting."
+                        case commands.FW.DEFAULT if is_author or source_author:
+                            message = "Waiting for CI to create followup forward-ports."
+                        case commands.FW.SKIPCI if is_reviewer or source_reviewer:
+                            message = "Not waiting for CI to create followup forward-ports."
+                        case commands.FW.SKIPMERGE if is_reviewer or source_reviewer:
+                            message = "Not waiting for merge to create followup forward-ports."
+                        case _:
+                            msg = f"you don't have the right to {command}."
 
-                                if ping and p == self:
-                                    msg = m
-                                else:
-                                    if ping:
-                                        m = f"@{login} {m}"
-                                    self.env['runbot_merge.pull_requests.feedback'].create({
-                                        'repository': p.repository.id,
-                                        'pull_request': p.number,
-                                        'message': m,
-                                    })
-                        else:
-                            msg = "you can't set a forward-port limit."
-                    elif source_reviewer or is_reviewer:
+                    if not msg:
                         (self.source_id or self).batch_id.fw_policy = command.name.lower()
-                        match command:
-                            case commands.FW.DEFAULT:
-                                message = "Waiting for CI to create followup forward-ports."
-                            case commands.FW.SKIPCI:
-                                message = "Not waiting for CI to create followup forward-ports."
-                            case commands.FW.SKIPMERGE:
-                                message = "Not waiting for merge to create followup forward-ports."
                         feedback(message=message)
-                    else:
-                        msg = "you can't configure forward-port CI."
                 case commands.Limit(branch) if is_author:
                     if branch is None:
                         feedback(message="'ignore' is deprecated, use 'fw=no' to disable forward porting.")
@@ -957,7 +943,7 @@ For your own safety I've ignored *everything in your entire comment*.
         if not self.source_id and self.state != 'merged':
             self.limit_id = limit_id
             if branch_key(limit_id) <= branch_key(self.target):
-                return False, "Forward-port disabled."
+                return False, "Forward-port disabled (via limit)."
             else:
                 return False, f"Forward-porting to {limit_id.name!r}."
 

From 3062f302459bb31d436fa591bd3bd763cc507e7b Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 5 Jul 2024 13:29:20 +0200
Subject: [PATCH 155/192] [IMP] runbot_merge: pass commit-tree message via
 stdin

Automating via parameters is riskier as we can hit the CLI
limitations (cf 0a839a4857e3c0d6999321efbbb6ed7c6b53a170). Going
through stdin is a lot safer and cleaner when automating, and it's not
much of an imposition here.
---
 runbot_merge/git.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/runbot_merge/git.py b/runbot_merge/git.py
index 771a6733..f44c1cd2 100644
--- a/runbot_merge/git.py
+++ b/runbot_merge/git.py
@@ -222,6 +222,7 @@ class Repo:
                 authorship['GIT_COMMITTER_DATE'] = committer[2]
 
         return self.with_config(
+            input=message,
             stdout=subprocess.PIPE,
             text=True,
             env={
@@ -235,7 +236,7 @@ class Repo:
         )._run(
             'commit-tree',
             tree,
-            '-m', message,
+            '-F', '-',
             *itertools.chain.from_iterable(('-p', p) for p in parents),
         )
 

From b1d3278de197e998458a3f8a6c83fcb56a623c9f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 5 Jul 2024 13:32:02 +0200
Subject: [PATCH 156/192] [CHG] forwardport: perform forward porting without
 working copies

The goal is to reduce maintenance and odd disk interactions &
concurrency issues, by not creating concurrent clones, not having to
push forks back in the repository, etc... it also removes the need to
cleanup "scratch" working copies though that looks not to have been an
issue in a while.

The work is done on isolated objects without using or mutating refs,
so even concurrent work should not be a problem.

This turns out to not be any more verbose (less so if anything) than
using `cherry-pick`, as that is not really designed for scripted /
non-interactive use, or for squashing commits thereafter. Working
directly with trees and commits is quite a bit cleaner even without a
ton of helpers.

Much of the credit goes to Julia Evans for [their investigation of
3-way merges as the underpinnings of cherry-picking][3-way merge],
this would have been a lot more difficult if I'd had to rediscover the
merge-base trick independently.

A few things have been changed by this:

- The old trace/stderr from cherrypick has disappeared as it's
  generated by cherrypick, but for a non-interactive use it's kinda
  useless anyway so I probably should have looked into removing it
  earlier (I think the main use was investigation of the inflateinit
  issue).
- Error on emptied commits has to be hand-rolled as `merge-tree`
  couldn't care less, this is not hard but is a bit annoying.
- `merge-tree`'s conflict information only references raw commits,
  which makes sense, but requires updating a bunch of tests. Then
  again so does the fact that it *usually* doesn't send anything to
  stderr, so that's usually disappearing.

Conveniently `merge-tree` merges the conflict marker directly in the
files / tree so we don't have to mess about moving them back out of
the repository and into the working copy as I assume cherry-pick does,
which means we don't have to try and commit them back in ether. That
is a huge part of the gain over faffing about with the working copy.

Fixes #847

[3-way merge]: https://jvns.ca/blog/2023/11/10/how-cherry-pick-and-revert-work/
---
 forwardport/models/forwardport.py             | 177 +++++++-------
 forwardport/models/project.py                 | 216 ++++++++----------
 forwardport/tests/test_batches.py             |   9 +-
 forwardport/tests/test_conflicts.py           |  15 +-
 forwardport/tests/test_updates.py             |  25 +-
 runbot_merge/git.py                           |  17 +-
 runbot_merge/models/batch.py                  |  11 +-
 runbot_merge/models/crons/git_maintenance.py  |   2 +-
 .../models/project_freeze/__init__.py         |  14 +-
 runbot_merge/models/stagings_create.py        |   6 +-
 10 files changed, 220 insertions(+), 272 deletions(-)

diff --git a/forwardport/models/forwardport.py b/forwardport/models/forwardport.py
index 54521621..a0604da7 100644
--- a/forwardport/models/forwardport.py
+++ b/forwardport/models/forwardport.py
@@ -161,94 +161,94 @@ class ForwardPortTasks(models.Model, Queue):
         #       only then create the PR objects
         # TODO: maybe do that after making forward-port WC-less, so all the
         #       branches can be pushed atomically at once
-        with contextlib.ExitStack() as s:
-            for descendant in self.batch_id.descendants():
-                target = pr._find_next_target()
-                if target is None:
-                    _logger.info("Will not forward-port %s: no next target", pr.display_name)
-                    return
+        for descendant in self.batch_id.descendants():
+            target = pr._find_next_target()
+            if target is None:
+                _logger.info("Will not forward-port %s: no next target", pr.display_name)
+                return
 
-                if PullRequests.search_count([
-                    ('source_id', '=', source.id),
-                    ('target', '=', target.id),
-                    ('state', 'not in', ('closed', 'merged')),
-                ]):
-                    _logger.warning("Will not forward-port %s: already ported", pr.display_name)
-                    return
+            if PullRequests.search_count([
+                ('source_id', '=', source.id),
+                ('target', '=', target.id),
+                ('state', 'not in', ('closed', 'merged')),
+            ]):
+                _logger.warning("Will not forward-port %s: already ported", pr.display_name)
+                return
 
-                if target != descendant.target:
-                    self.env['runbot_merge.pull_requests.feedback'].create({
-                        'repository': repository.id,
-                        'pull_request': source.id,
-                        'token_field': 'fp_github_token',
-                        'message': """\
+            if target != descendant.target:
+                self.env['runbot_merge.pull_requests.feedback'].create({
+                    'repository': repository.id,
+                    'pull_request': source.id,
+                    'token_field': 'fp_github_token',
+                    'message': """\
 {pr.ping}unable to port this PR forwards due to inconsistency: goes from \
 {pr.target.name} to {next_target.name} but {batch} ({batch_prs}) targets \
 {batch.target.name}.
 """.format(pr=pr, next_target=target, batch=descendant, batch_prs=', '.join(descendant.mapped('prs.display_name')))
-                    })
-                    return
-
-                ref = descendant.prs[:1].refname
-                # NOTE: ports the new source everywhere instead of porting each
-                #       PR to the next step as it does not *stop* on conflict
-                conflict, working_copy = source._create_fp_branch(target, ref, s)
-                working_copy.push('target', ref)
-
-                remote_target = repository.fp_remote_target
-                owner, _ = remote_target.split('/', 1)
-                message = source.message + f"\n\nForward-Port-Of: {pr.display_name}"
-
-                title, body = re.match(r'(?P<title>[^\n]+)\n*(?P<body>.*)', message, flags=re.DOTALL).groups()
-                r = gh.post(f'https://api.github.com/repos/{pr.repository.name}/pulls', json={
-                    'base': target.name,
-                    'head': f'{owner}:{ref}',
-                    'title': '[FW]' + (' ' if title[0] != '[' else '') + title,
-                    'body': body
                 })
-                if not r.ok:
-                    _logger.warning("Failed to create forward-port PR for %s, deleting branches", pr.display_name)
-                    # delete all the branches this should automatically close the
-                    # PRs if we've created any. Using the API here is probably
-                    # simpler than going through the working copies
-                    d = gh.delete(f'https://api.github.com/repos/{remote_target}/git/refs/heads/{ref}')
-                    if d.ok:
-                        _logger.info("Deleting %s:%s=success", remote_target, ref)
-                    else:
-                        _logger.warning("Deleting %s:%s=%s", remote_target, ref, d.text)
-                    raise RuntimeError(f"Forwardport failure: {pr.display_name} ({r.text})")
+                return
 
-                new_pr = PullRequests._from_gh(r.json())
-                _logger.info("Created forward-port PR %s", new_pr)
-                new_pr.write({
-                    'batch_id': descendant.id, # should already be set correctly but...
-                    'merge_method': pr.merge_method,
-                    'source_id': source.id,
-                    # only link to previous PR of sequence if cherrypick passed
-                    # FIXME: apply parenting of siblings? Apply parenting *to* siblings?
-                    'parent_id': pr.id if not conflict else False,
-                    'detach_reason': "{1}\n{2}".format(*conflict).strip() if conflict else None,
-                })
+            ref = descendant.prs[:1].refname
+            # NOTE: ports the new source everywhere instead of porting each
+            #       PR to the next step as it does not *stop* on conflict
+            repo = git.get_local(source.repository)
+            conflict, head = source._create_fp_branch(repo, target)
+            repo.push(git.fw_url(pr.repository), f'{head}:refs/heads/{ref}')
 
-                if conflict:
-                    self.env.ref('runbot_merge.forwardport.failure.conflict')._send(
-                        repository=pr.repository,
-                        pull_request=pr.number,
-                        token_field='fp_github_token',
-                        format_args={'source': source, 'pr': pr, 'new': new_pr, 'footer': FOOTER},
-                    )
-                new_pr._fp_conflict_feedback(pr, {pr: conflict})
+            remote_target = repository.fp_remote_target
+            owner, _ = remote_target.split('/', 1)
+            message = source.message + f"\n\nForward-Port-Of: {pr.display_name}"
 
-                labels = ['forwardport']
-                if conflict:
-                    labels.append('conflict')
-                self.env['runbot_merge.pull_requests.tagging'].create({
-                    'repository': new_pr.repository.id,
-                    'pull_request': new_pr.number,
-                    'tags_add': labels,
-                })
+            title, body = re.match(r'(?P<title>[^\n]+)\n*(?P<body>.*)', message, flags=re.DOTALL).groups()
+            r = gh.post(f'https://api.github.com/repos/{pr.repository.name}/pulls', json={
+                'base': target.name,
+                'head': f'{owner}:{ref}',
+                'title': '[FW]' + (' ' if title[0] != '[' else '') + title,
+                'body': body
+            })
+            if not r.ok:
+                _logger.warning("Failed to create forward-port PR for %s, deleting branches", pr.display_name)
+                # delete all the branches this should automatically close the
+                # PRs if we've created any. Using the API here is probably
+                # simpler than going through the working copies
+                d = gh.delete(f'https://api.github.com/repos/{remote_target}/git/refs/heads/{ref}')
+                if d.ok:
+                    _logger.info("Deleting %s:%s=success", remote_target, ref)
+                else:
+                    _logger.warning("Deleting %s:%s=%s", remote_target, ref, d.text)
+                raise RuntimeError(f"Forwardport failure: {pr.display_name} ({r.text})")
 
-                pr = new_pr
+            new_pr = PullRequests._from_gh(r.json())
+            _logger.info("Created forward-port PR %s", new_pr)
+            new_pr.write({
+                'batch_id': descendant.id, # should already be set correctly but...
+                'merge_method': pr.merge_method,
+                'source_id': source.id,
+                # only link to previous PR of sequence if cherrypick passed
+                # FIXME: apply parenting of siblings? Apply parenting *to* siblings?
+                'parent_id': pr.id if not conflict else False,
+                'detach_reason': "{1}\n{2}".format(*conflict).strip() if conflict else None,
+            })
+
+            if conflict:
+                self.env.ref('runbot_merge.forwardport.failure.conflict')._send(
+                    repository=pr.repository,
+                    pull_request=pr.number,
+                    token_field='fp_github_token',
+                    format_args={'source': source, 'pr': pr, 'new': new_pr, 'footer': FOOTER},
+                )
+            new_pr._fp_conflict_feedback(pr, {pr: conflict})
+
+            labels = ['forwardport']
+            if conflict:
+                labels.append('conflict')
+            self.env['runbot_merge.pull_requests.tagging'].create({
+                'repository': new_pr.repository.id,
+                'pull_request': new_pr.number,
+                'tags_add': labels,
+            })
+
+            pr = new_pr
 
 class UpdateQueue(models.Model, Queue):
     _name = 'forwardport.updates'
@@ -286,8 +286,9 @@ class UpdateQueue(models.Model, Queue):
                     )
                     return
 
-                conflicts, working_copy = previous._create_fp_branch(
-                    child.target, child.refname, s)
+                repo = git.get_local(previous.repository)
+                conflicts, new_head = previous._create_fp_branch(repo, child.target)
+
                 if conflicts:
                     _, out, err, _ = conflicts
                     self.env.ref('runbot_merge.forwardport.updates.conflict.parent')._send(
@@ -308,9 +309,8 @@ class UpdateQueue(models.Model, Queue):
                         },
                     )
 
-                new_head = working_copy.stdout().rev_parse(child.refname).stdout.decode().strip()
-                commits_count = int(working_copy.stdout().rev_list(
-                    f'{child.target.name}..{child.refname}',
+                commits_count = int(repo.stdout().rev_list(
+                    f'{child.target.name}..{new_head}',
                     count=True
                 ).stdout.decode().strip())
                 old_head = child.head
@@ -320,16 +320,11 @@ class UpdateQueue(models.Model, Queue):
                     # 'state': 'opened',
                     'squash': commits_count == 1,
                 })
-                # push the new head to the local cache: in some cases github
-                # doesn't propagate revisions fast enough so on the next loop we
-                # can't find the revision we just pushed
-                dummy_branch = str(uuid.uuid4())
-                ref = git.get_local(previous.repository, 'fp_github')
-                working_copy.push(ref._directory, f'{new_head}:refs/heads/{dummy_branch}')
-                ref.branch('--delete', '--force', dummy_branch)
                 # then update the child's branch to the new head
-                working_copy.push(f'--force-with-lease={child.refname}:{old_head}',
-                                  'target', child.refname)
+                repo.push(
+                    f'--force-with-lease={child.refname}:{old_head}',
+                    git.fw_url(child.repository),
+                    f"{new_head}:refs/heads/{child.refname}")
 
                 # committing here means github could technically trigger its
                 # webhook before sending a response, but committing before
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 5c664c25..31e0ec9c 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -19,9 +19,7 @@ import json
 import logging
 import operator
 import subprocess
-import tempfile
 import typing
-from pathlib import Path
 
 import dateutil.relativedelta
 import requests
@@ -30,7 +28,6 @@ from odoo import models, fields, api
 from odoo.exceptions import UserError
 from odoo.osv import expression
 from odoo.tools.misc import topological_sort, groupby
-from odoo.tools.appdirs import user_cache_dir
 from odoo.addons.base.models.res_partner import Partner
 from odoo.addons.runbot_merge import git, utils
 from odoo.addons.runbot_merge.models.pull_requests import Branch
@@ -289,17 +286,11 @@ class PullRequests(models.Model):
         return sorted(commits, key=lambda c: idx[c['sha']])
 
 
-    def _create_fp_branch(self, target_branch, fp_branch_name, cleanup):
+    def _create_fp_branch(self, source, target_branch):
         """ Creates a forward-port for the current PR to ``target_branch`` under
         ``fp_branch_name``.
 
         :param target_branch: the branch to port forward to
-        :param fp_branch_name: the name of the branch to create the FP under
-        :param ExitStack cleanup: so the working directories can be cleaned up
-        :return: A pair of an optional conflict information and a repository. If
-                 present the conflict information is composed of the hash of the
-                 conflicting commit, the stderr and stdout of the failed
-                 cherrypick and a list of all PR commit hashes
         :rtype: (None | (str, str, str, list[commit]), Repo)
         """
         logger = _logger.getChild(str(self.id))
@@ -308,104 +299,52 @@ class PullRequests(models.Model):
             "Forward-porting %s (%s) to %s",
             self.display_name, root.display_name, target_branch.name
         )
-        source = git.get_local(self.repository, 'fp_github')
-        r = source.with_config(stdout=subprocess.PIPE, stderr=subprocess.STDOUT).fetch()
-        logger.info("Updated cache repo %s:\n%s", source._directory, r.stdout.decode())
+        tree = source.with_config(stdout=subprocess.PIPE, stderr=subprocess.STDOUT).fetch()
+        logger.info("Updated cache repo %s:\n%s", source._directory, tree.stdout.decode())
 
-        logger.info("Create working copy...")
-        cache_dir = user_cache_dir('forwardport')
-        # PullRequest.display_name is `owner/repo#number`, so `owner` becomes a
-        # directory, `TemporaryDirectory` only creates the leaf, so we need to
-        # make sure `owner` exists in `cache_dir`.
-        Path(cache_dir, root.repository.name).parent.mkdir(parents=True, exist_ok=True)
-        working_copy = source.clone(
-            cleanup.enter_context(
-                tempfile.TemporaryDirectory(
-                    prefix=f'{root.display_name}-to-{target_branch.name}',
-                    dir=cache_dir)),
-            branch=target_branch.name
-        )
-
-        r = working_copy.with_config(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) \
-            .fetch(git.source_url(self.repository, 'fp_github'), root.head)
+        tree = source.with_config(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) \
+            .fetch(git.source_url(self.repository), root.head)
         logger.info(
-            "Fetched head of %s into %s:\n%s",
+            "Fetched head of %s (%s):\n%s",
             root.display_name,
-            working_copy._directory,
-            r.stdout.decode()
+            root.head,
+            tree.stdout.decode()
         )
-        if working_copy.check(False).cat_file(e=root.head).returncode:
+        if source.check(False).cat_file(e=root.head).returncode:
             raise ForwardPortError(
                 f"During forward port of {self.display_name}, unable to find "
                 f"expected head of {root.display_name} ({root.head})"
             )
 
-        project_id = self.repository.project_id
-        # add target remote
-        working_copy.remote(
-            'add', 'target',
-            'https://{p.fp_github_token}@github.com/{r.fp_remote_target}'.format(
-                r=self.repository,
-                p=project_id
-            )
-        )
-        logger.info("Create FP branch %s in %s", fp_branch_name, working_copy._directory)
-        working_copy.checkout(b=fp_branch_name)
-
         try:
-            root._cherry_pick(working_copy)
-            return None, working_copy
+            return None, root._cherry_pick(source, target_branch.name)
         except CherrypickError as e:
             h, out, err, commits = e.args
 
-            # using git diff | git apply -3 to get the entire conflict set
-            # turns out to not work correctly: in case files have been moved
-            # / removed (which turns out to be a common source of conflicts
-            # when forward-porting) it'll just do nothing to the working copy
-            # so the "conflict commit" will be empty
-            # switch to a squashed-pr branch
-            working_copy.check(True).checkout('-bsquashed', root.head)
             # commits returns oldest first, so youngest (head) last
             head_commit = commits[-1]['commit']
 
             to_tuple = operator.itemgetter('name', 'email')
-            def to_dict(term, vals):
-                return {'GIT_%s_NAME' % term: vals[0], 'GIT_%s_EMAIL' % term: vals[1], 'GIT_%s_DATE' % term: vals[2]}
             authors, committers = set(), set()
-            for c in (c['commit'] for c in commits):
-                authors.add(to_tuple(c['author']))
-                committers.add(to_tuple(c['committer']))
-            fp_authorship = (project_id.fp_github_name, '', '')
+            for commit in (c['commit'] for c in commits):
+                authors.add(to_tuple(commit['author']))
+                committers.add(to_tuple(commit['committer']))
+            fp_authorship = (self.repository.project_id.fp_github_name, '', '')
             author = fp_authorship if len(authors) != 1 \
                 else authors.pop() + (head_commit['author']['date'],)
             committer = fp_authorship if len(committers) != 1 \
                 else committers.pop() + (head_commit['committer']['date'],)
-            conf = working_copy.with_config(env={
-                **to_dict('AUTHOR', author),
-                **to_dict('COMMITTER', committer),
-                'GIT_COMMITTER_DATE': '',
-            })
-            # squash to a single commit
-            conf.reset('--soft', commits[0]['parents'][0]['sha'])
-            conf.commit(a=True, message="temp")
-            squashed = conf.stdout().rev_parse('HEAD').stdout.strip().decode()
-
-            # switch back to the PR branch
-            conf.checkout(fp_branch_name)
-            # cherry-pick the squashed commit to generate the conflict
-            conf.with_params(
+            conf = source.with_params(
                 'merge.renamelimit=0',
                 'merge.renames=copies',
                 'merge.conflictstyle=zdiff3'
-            ) \
-                .with_config(check=False) \
-                .cherry_pick(squashed, no_commit=True, strategy="ort")
-            status = conf.stdout().status(short=True, untracked_files='no').stdout.decode()
-            if err.strip():
-                err = err.rstrip() + '\n----------\nstatus:\n' + status
-            else:
-                err = 'status:\n' + status
+            ).with_config(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
+            tree = conf.with_config(check=False).merge_tree(
+                '--merge-base', commits[0]['parents'][0]['sha'],
+                target_branch.name,
+                root.head,
+            )
             # if there was a single commit, reuse its message when committing
             # the conflict
             if len(commits) == 1:
@@ -421,77 +360,107 @@ stderr:
 {err}
 """
 
-            conf.with_config(input=str(msg).encode()) \
-                .commit(all=True, allow_empty=True, file='-')
+            target_head = source.stdout().rev_parse(target_branch.name).stdout.decode().strip()
+            commit = conf.commit_tree(
+                tree=tree.stdout.decode().splitlines(keepends=False)[0],
+                parents=[target_head],
+                message=str(msg),
+                author=author,
+                committer=committer[:2],
+            )
+            assert commit.returncode == 0,\
+                f"commit failed\n\n{commit.stdout.decode()}\n\n{commit.stderr.decode}"
+            hh = commit.stdout.strip()
 
-            return (h, out, err, [c['sha'] for c in commits]), working_copy
+            return (h, out, err, [c['sha'] for c in commits]), hh
 
-    def _cherry_pick(self, working_copy):
-        """ Cherrypicks ``self`` into the working copy
+    def _cherry_pick(self, repo: git.Repo, branch: Branch) -> str:
+        """ Cherrypicks ``self`` into ``branch``
 
-        :return: ``True`` if the cherrypick was successful, ``False`` otherwise
+        :return: the HEAD of the forward-port is successful
+        :raises CherrypickError: in case of conflict
         """
         # <xxx>.cherrypick.<number>
         logger = _logger.getChild(str(self.id)).getChild('cherrypick')
 
-        # original head so we can reset
-        original_head = working_copy.stdout().rev_parse('HEAD').stdout.decode().strip()
+        # target's head
+        head = repo.stdout().rev_parse(branch).stdout.decode().strip()
 
         commits = self.commits()
-        logger.info("%s: copy %s commits to %s\n%s", self, len(commits), original_head, '\n'.join(
-            '- %s (%s)' % (c['sha'], c['commit']['message'].splitlines()[0])
-            for c in commits
-        ))
+        logger.info(
+            "%s: copy %s commits to %s (%s)%s",
+            self, len(commits), branch, head, ''.join(
+                '\n- %s: %s' % (c['sha'], c['commit']['message'].splitlines()[0])
+                for c in commits
+            )
+        )
 
-        conf_base = working_copy.with_params(
+        conf = repo.with_params(
             'merge.renamelimit=0',
             'merge.renames=copies',
         ).with_config(
             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-            check=False
+            check=False,
         )
         for commit in commits:
             commit_sha = commit['sha']
-            # config (global -c) or commit options don't really give access to
-            # setting dates
-            cm = commit['commit'] # get the "git" commit object rather than the "github" commit resource
-            env = {
-                'GIT_AUTHOR_NAME': cm['author']['name'],
-                'GIT_AUTHOR_EMAIL': cm['author']['email'],
-                'GIT_AUTHOR_DATE': cm['author']['date'],
-                'GIT_COMMITTER_NAME': cm['committer']['name'],
-                'GIT_COMMITTER_EMAIL': cm['committer']['email'],
-            }
-            configured = working_copy.with_config(env=env)
+            # merge-tree is a bit stupid and gets confused when the options
+            # follow the parameters
+            r = conf.merge_tree('--merge-base', commit['parents'][0]['sha'], head, commit_sha)
+            new_tree = r.stdout.decode().splitlines(keepends=False)[0]
+            if r.returncode:
+                # For merge-tree the stdout on conflict is of the form
+                #
+                # oid of toplevel tree
+                # conflicted file info+
+                #
+                # informational messages+
+                #
+                # to match cherrypick we only want the informational messages,
+                # so strip everything else
+                r.stdout = r.stdout.split(b'\n\n')[-1]
+            else:
+                # By default cherry-pick fails if a non-empty commit becomes
+                # empty (--empty=stop), also it fails when cherrypicking already
+                # empty commits which I didn't think we prevented but clearly we
+                # do...?
+                parent_tree = conf.rev_parse(f'{head}^{{tree}}').stdout.decode().strip()
+                if parent_tree == new_tree:
+                    r.returncode = 1
+                    r.stdout = f"You are currently cherry-picking commit {commit_sha}.".encode()
+                    r.stderr = b"The previous cherry-pick is now empty, possibly due to conflict resolution."
 
-            conf = conf_base.with_config(env={**env, 'GIT_TRACE': 'true'})
-            # first try with default / low renamelimit
-            r = conf.cherry_pick(commit_sha, strategy="ort")
             logger.debug("Cherry-picked %s: %s\n%s\n%s", commit_sha, r.returncode, r.stdout.decode(), _clean_rename(r.stderr.decode()))
-
-            if r.returncode: # pick failed, reset and bail
+            if r.returncode: # pick failed, bail
                 # try to log inflateInit: out of memory errors as warning, they
-                # seem to return the status code 128
+                # seem to return the status code 128 (nb: may not work anymore
+                # with merge-tree, idk)
                 logger.log(
                     logging.WARNING if r.returncode == 128 else logging.INFO,
                     "forward-port of %s (%s) failed at %s",
                     self, self.display_name, commit_sha)
-                configured.reset('--hard', original_head)
+
                 raise CherrypickError(
                     commit_sha,
                     r.stdout.decode(),
                     _clean_rename(r.stderr.decode()),
                     commits
                 )
+            # get the "git" commit object rather than the "github" commit resource
+            cc = conf.commit_tree(
+                tree=new_tree,
+                parents=[head],
+                message=str(self._make_fp_message(commit)),
+                author=map_author(commit['commit']['author']),
+                committer=map_committer(commit['commit']['committer']),
+            )
+            if cc.returncode:
+                raise CherrypickError(commit_sha, cc.stdout.decode(), cc.stderr.decode(), commits)
 
-            msg = self._make_fp_message(commit)
+            head = cc.stdout.strip()
+            logger.info('%s -> %s', commit_sha, head)
 
-            # replace existing commit message with massaged one
-            configured \
-                .with_config(input=str(msg).encode())\
-                .commit(amend=True, file='-')
-            result = configured.stdout().rev_parse('HEAD').stdout.decode()
-            logger.info('%s: success -> %s', commit_sha, result)
+        return head
 
     def _make_fp_message(self, commit):
         cmap = json.loads(self.commits_map)
@@ -541,6 +510,11 @@ stderr:
                     format_args={'pr': pr, 'source': source},
                 )
 
+
+map_author = operator.itemgetter('name', 'email', 'date')
+map_committer = operator.itemgetter('name', 'email')
+
+
 class Stagings(models.Model):
     _inherit = 'runbot_merge.stagings'
 
diff --git a/forwardport/tests/test_batches.py b/forwardport/tests/test_batches.py
index b0b7155a..2ecab8b6 100644
--- a/forwardport/tests/test_batches.py
+++ b/forwardport/tests/test_batches.py
@@ -381,7 +381,7 @@ def test_add_to_forward_port_conflict(env, config, make_repo, users):
     assert pr2_c.comments == [
         seen(env, pr2_c, users),
         # should have conflicts
-        (users['user'], matches("""@{user} cherrypicking of pull request {previous.display_name} failed.
+        (users['user'], """@{user} cherrypicking of pull request {previous.display_name} failed.
 
 stdout:
 ```
@@ -390,11 +390,6 @@ CONFLICT (add/add): Merge conflict in b
 
 ```
 
-stderr:
-```
-$$
-```
-
 Either perform the forward-port manually (and push to this branch, proceeding as usual) or close this PR (maybe?).
 
 In the former case, you may want to edit this PR message as well.
@@ -402,5 +397,5 @@ In the former case, you may want to edit this PR message as well.
 :warning: after resolving this conflict, you will need to merge it via @{project.github_prefix}.
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
-""".format(project=project, previous=pr2_b_id, **users)))
+""".format(project=project, previous=pr2_b_id, **users))
     ]
diff --git a/forwardport/tests/test_conflicts.py b/forwardport/tests/test_conflicts.py
index 030e534b..7011b1b3 100644
--- a/forwardport/tests/test_conflicts.py
+++ b/forwardport/tests/test_conflicts.py
@@ -62,17 +62,17 @@ def test_conflict(env, config, make_repo, users):
     assert prod.read_tree(c) == {
         'f': 'c',
         'g': 'a',
-        'h': matches('''<<<\x3c<<< HEAD
+        'h': matches('''<<<\x3c<<< $$
 a
-||||||| parent of $$ (temp)
+||||||| $$
 =======
 xxx
->>>\x3e>>> $$ (temp)
+>>>\x3e>>> $$
 '''),
     }
     assert prc.comments == [
         seen(env, prc, users),
-        (users['user'], matches(
+        (users['user'],
 f'''@{users['user']} @{users['reviewer']} cherrypicking of pull request {pra_id.display_name} failed.
 
 stdout:
@@ -82,11 +82,6 @@ CONFLICT (add/add): Merge conflict in h
 
 ```
 
-stderr:
-```
-$$
-```
-
 Either perform the forward-port manually (and push to this branch, proceeding as usual) or close this PR (maybe?).
 
 In the former case, you may want to edit this PR message as well.
@@ -94,7 +89,7 @@ In the former case, you may want to edit this PR message as well.
 :warning: after resolving this conflict, you will need to merge it via @{project.github_prefix}.
 
 More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
-'''))
+''')
     ]
 
     prb = prod.get_pr(prb_id.number)
diff --git a/forwardport/tests/test_updates.py b/forwardport/tests/test_updates.py
index d68c6c92..b13cd16a 100644
--- a/forwardport/tests/test_updates.py
+++ b/forwardport/tests/test_updates.py
@@ -445,12 +445,12 @@ def test_subsequent_conflict(env, make_repo, config, users):
     assert repo.read_tree(repo.commit(pr3_id.head)) == {
         'f': 'c',
         'g': 'a',
-        'h': matches('''<<<\x3c<<< HEAD
+        'h': matches('''<<<\x3c<<< $$
 a
-||||||| parent of $$ (temp)
+||||||| $$
 =======
 conflict!
->>>\x3e>>> $$ (temp)
+>>>\x3e>>> $$
 '''),
         'x': '0',
     }
@@ -470,7 +470,7 @@ conflict!
     # 1. link to status page
     # 2. forward-port chain thing
     assert repo.get_pr(pr3_id.number).comments[2:] == [
-        (users['user'], matches(f'''\
+        (users['user'], f'''\
 @{users['user']} @{users['reviewer']} WARNING: the update of {pr2_id.display_name} to {pr2_id.head} has caused a \
 conflict in this pull request, data may have been lost.
 
@@ -478,20 +478,5 @@ stdout:
 ```
 Auto-merging h
 CONFLICT (add/add): Merge conflict in h
-```
-
-stderr:
-```
-$$ trace: built-in: git cherry-pick {pr2_id.head} --strategy ort
-error: could not apply {pr2_id.head[:7]}... newfiles
-hint: After resolving the conflicts, mark them with
-hint: "git add/rm <pathspec>", then run
-hint: "git cherry-pick --continue".
-hint: You can instead skip this commit with "git cherry-pick --skip".
-hint: To abort and get back to the state before "git cherry-pick",
-hint: run "git cherry-pick --abort".
-hint: Disable this message with "git config advice.mergeConflict false"
-----------
-status:
-```'''))
+```'''),
     ]
diff --git a/runbot_merge/git.py b/runbot_merge/git.py
index f44c1cd2..caaa6e5d 100644
--- a/runbot_merge/git.py
+++ b/runbot_merge/git.py
@@ -13,16 +13,21 @@ from .github import MergeError, PrCommit
 
 _logger = logging.getLogger(__name__)
 
-
-def source_url(repository, prefix: str) -> str:
+def source_url(repository) -> str:
     return 'https://{}@github.com/{}'.format(
-        repository.project_id[f'{prefix}_token'],
+        repository.project_id.github_token,
         repository.name,
     )
 
+def fw_url(repository) -> str:
+    return 'https://{}@github.com/{}'.format(
+        repository.project_id.fp_github_token,
+        repository.fp_remote_target,
+    )
+
 Authorship = Union[Tuple[str, str], Tuple[str, str, str]]
 
-def get_local(repository, prefix: Optional[str]) -> 'Optional[Repo]':
+def get_local(repository, *, clone: bool = True) -> 'Optional[Repo]':
     repos_dir = pathlib.Path(user_cache_dir('mergebot'))
     repos_dir.mkdir(parents=True, exist_ok=True)
     # NB: `repository.name` is `$org/$name` so this will be a subdirectory, probably
@@ -30,9 +35,9 @@ def get_local(repository, prefix: Optional[str]) -> 'Optional[Repo]':
 
     if repo_dir.is_dir():
         return git(repo_dir)
-    elif prefix:
+    elif clone:
         _logger.info("Cloning out %s to %s", repository.name, repo_dir)
-        subprocess.run(['git', 'clone', '--bare', source_url(repository, prefix), str(repo_dir)], check=True)
+        subprocess.run(['git', 'clone', '--bare', source_url(repository), str(repo_dir)], check=True)
         # bare repos don't have fetch specs by default, and fetching *into*
         # them is a pain in the ass, configure fetch specs so `git fetch`
         # works properly
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index c85eb4f7..de70d67b 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -13,7 +13,7 @@ from psycopg2 import sql
 
 from odoo import models, fields, api
 from .utils import enum
-
+from .. import git
 
 _logger = logging.getLogger(__name__)
 FOOTER = '\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n'
@@ -330,12 +330,11 @@ class Batch(models.Model):
             base64.urlsafe_b64encode(os.urandom(3)).decode()
         )
         conflicts = {}
-        with contextlib.ExitStack() as s:
-            for pr in prs:
-                conflicts[pr], working_copy = pr._create_fp_branch(
-                    target, new_branch, s)
+        for pr in prs:
+            repo = git.get_local(pr.repository)
+            conflicts[pr], head = pr._create_fp_branch(repo, target)
 
-                working_copy.push('target', new_branch)
+            repo.push(git.fw_url(pr.repository), f"{head}:refs/heads/{new_branch}")
 
         gh = requests.Session()
         gh.headers['Authorization'] = 'token %s' % proj.fp_github_token
diff --git a/runbot_merge/models/crons/git_maintenance.py b/runbot_merge/models/crons/git_maintenance.py
index 2b0e2182..29dd028a 100644
--- a/runbot_merge/models/crons/git_maintenance.py
+++ b/runbot_merge/models/crons/git_maintenance.py
@@ -24,7 +24,7 @@ class GC(models.TransientModel):
 
         # run on all repos with a forwardport target (~ forwardport enabled)
         for repo in self.env['runbot_merge.repository'].search([]):
-            repo_git = get_local(repo, prefix=None)
+            repo_git = get_local(repo, clone=False)
             if not repo_git:
                 continue
 
diff --git a/runbot_merge/models/project_freeze/__init__.py b/runbot_merge/models/project_freeze/__init__.py
index 3c645969..0a39ceae 100644
--- a/runbot_merge/models/project_freeze/__init__.py
+++ b/runbot_merge/models/project_freeze/__init__.py
@@ -213,15 +213,15 @@ class FreezeWizard(models.Model):
 
         gh_sessions = {r: r.github() for r in self.project_id.repo_ids}
         repos: Dict[Repository, git.Repo] = {
-            r: git.get_local(r, 'github').check(False)
+            r: git.get_local(r).check(False)
             for r in self.project_id.repo_ids
         }
         for repo, copy in repos.items():
-            copy.fetch(git.source_url(repo, 'github'), '+refs/heads/*:refs/heads/*')
+            copy.fetch(git.source_url(repo), '+refs/heads/*:refs/heads/*')
         all_prs = self.release_pr_ids.pr_id | self.bump_pr_ids.pr_id
         for pr in all_prs:
             repos[pr.repository].fetch(
-                git.source_url(pr.repository, 'github'),
+                git.source_url(pr.repository),
                 pr.head,
             )
 
@@ -282,7 +282,7 @@ class FreezeWizard(models.Model):
             repo_id = rel.repository_id
 
             if repos[repo_id].push(
-                git.source_url(repo_id, 'github'),
+                git.source_url(repo_id),
                 f'{rel_heads[repo_id]}:refs/heads/{self.branch_name}',
             ).returncode:
                 failure = ('create', repo_id.name, self.branch_name)
@@ -294,7 +294,7 @@ class FreezeWizard(models.Model):
             for bump in self.bump_pr_ids:
                 repo_id = bump.repository_id
                 if repos[repo_id].push(
-                    git.source_url(repo_id, 'github'),
+                    git.source_url(repo_id),
                     f'{bump_heads[repo_id]}:refs/heads/{master_name}'
                 ).returncode:
                     failure = ('fast-forward', repo_id.name, master_name)
@@ -310,7 +310,7 @@ class FreezeWizard(models.Model):
             for prev_id in to_revert:
                 if repos[prev_id].push(
                     '-f',
-                    git.source_url(prev_id, 'github'),
+                    git.source_url(prev_id),
                     f'{prevs[prev_id]}:refs/heads/{master_name}',
                 ).returncode:
                     failures.append(prev_id.name)
@@ -323,7 +323,7 @@ class FreezeWizard(models.Model):
 
             for prev_id in to_delete:
                 if repos[prev_id].push(
-                    git.source_url(prev_id, 'github'),
+                    git.source_url(prev_id),
                     f':refs/heads/{self.branch_name}'
                 ).returncode:
                     failures.append(prev_id.name)
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index ddc9c3d4..2005416b 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -177,7 +177,7 @@ For-Commit-Id: {it.head}
         )
         it.repo.stdout(False).check(True).push(
             '-f',
-            git.source_url(repo, 'github'),
+            git.source_url(repo),
             f'{it.head}:refs/heads/staging.{branch.name}',
         )
 
@@ -231,9 +231,9 @@ def staging_setup(
         gh = repo.github()
         head = gh.head(target.name)
 
-        source = git.get_local(repo, 'github')
+        source = git.get_local(repo)
         source.fetch(
-            git.source_url(repo, 'github'),
+            git.source_url(repo),
             # a full refspec is necessary to ensure we actually fetch the ref
             # (not just the commit it points to) and update it.
             # `git fetch $remote $branch` seems to work locally, but it might

From 02013a53d9eff756c41ce16ee819a591384355c9 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 9 Jul 2024 14:06:19 +0200
Subject: [PATCH 157/192] [IMP] runbot_merge: message when approving a PR in
 error

I thought I'd removed the error message when approving an already
approved PR but apparently not?

However we can improve the message in that specific case, to make the
expected operation clearer.

Fixes #906
---
 runbot_merge/models/pull_requests.py |  7 +++-
 runbot_merge/tests/test_basic.py     | 49 +++++++++++++++-------------
 2 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 75c76880..5602f0be 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1052,10 +1052,15 @@ For your own safety I've ignored *everything in your entire comment*.
         if not newstate:
             # Don't fail the entire command if someone tries to approve an
             # already-approved PR.
+            if self.error:
+                msg = "This PR is already reviewed, it's in error, you might want to `retry` it instead " \
+                      "(if you have already confirmed the error is not legitimate)."
+            else:
+                msg = "This PR is already reviewed, reviewing it again is useless."
             self.env['runbot_merge.pull_requests.feedback'].create({
                 'repository': self.repository.id,
                 'pull_request': self.number,
-                'message': "This PR is already reviewed, reviewing it again is useless.",
+                'message': msg,
             })
             return None
 
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 9133e48b..3b982984 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -1199,33 +1199,41 @@ class TestRetry:
         reviewer asks for it
         """
         with repo:
-            prx = _simple_init(repo)
-            repo.post_status(prx.head, 'success', 'ci/runbot')
-            repo.post_status(prx.head, 'success', 'legal/cla')
-            prx.post_comment('hansen r+ delegate=%s rebase-merge' % users['other'],
-                             config["role_reviewer"]['token'])
+            pr = _simple_init(repo)
+            repo.post_status(pr.head, 'success', 'ci/runbot')
+            repo.post_status(pr.head, 'success', 'legal/cla')
+            pr.post_comment(f'hansen r+ delegate={users["other"]} rebase-merge',
+                            config["role_reviewer"]['token'])
         env.run_crons()
-        assert env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number)
-        ]).staging_id
+        pr_id = to_pr(env, pr)
+        assert pr_id.staging_id
 
         staging_head = repo.commit('heads/staging.master')
         with repo:
             repo.post_status('staging.master', 'success', 'legal/cla')
             repo.post_status('staging.master', 'failure', 'ci/runbot')
         env.run_crons()
-        assert env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number)
-        ]).state == 'error'
+        assert pr_id.state == 'error'
 
         with repo:
-            prx.post_comment('hansen retry', config['role_' + retrier]['token'])
-        assert env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number)
-        ]).state == 'ready'
+            pr.post_comment('hansen r+ rebase-ff', config["role_reviewer"]['token'])
+        env.run_crons()
+        assert pr_id.state == 'error'
+        assert pr.comments == [
+            (users['reviewer'], f'hansen r+ delegate={users["other"]} rebase-merge'),
+            seen(env, pr, users),
+            (users['user'], 'Merge method set to rebase and merge, using the PR as merge commit message.'),
+            (users['user'], '@{user} @{reviewer} staging failed: ci/runbot'.format_map(users)),
+            (users['reviewer'], 'hansen r+ rebase-ff'),
+            (users['user'], "This PR is already reviewed, it's in error, you might want to `retry` it instead "
+                            "(if you have already confirmed the error is not legitimate)."),
+            (users['user'], 'Merge method set to rebase and fast-forward.'),
+        ]
+        assert pr_id.merge_method == 'rebase-ff'
+
+        with repo:
+            pr.post_comment('hansen retry', config['role_' + retrier]['token'])
+        assert pr_id.state == 'ready'
         env.run_crons('runbot_merge.merge_cron', 'runbot_merge.staging_cron')
 
         staging_head2 = repo.commit('heads/staging.master')
@@ -1234,10 +1242,7 @@ class TestRetry:
             repo.post_status('staging.master', 'success', 'legal/cla')
             repo.post_status('staging.master', 'success', 'ci/runbot')
         env.run_crons()
-        assert env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number)
-        ]).state == 'merged'
+        assert pr_id.state == 'merged'
 
     def test_retry_again_message(self, env, repo, users, config, page):
         """ For a retried PR, the error message on the PR's page should be the

From d6bb18e358a779ae9cb8e0fca4450a3caf1c8bfc Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 11 Jul 2024 11:53:01 +0200
Subject: [PATCH 158/192] [ADD] runbot_merge: rendering of PR descriptions

Previously PR descriptions were displayed as raw text in the PR
dashboard. While not wrong per se, this was pretty ugly and not always
convenient as e.g. links had to be copied by hand.

Push descriptions through pymarkdown for rendering them, with a few
customisations:

- Enabled footnotes & tables & fenced code blocks because GFM has
  that, this doesn't quite put pymarkdown's base behaviour on par with
  gfm (and py-gfm ultimately gave up on that effort moving to just
  wrap github's own markdown renderer instead).
- Don't allow raw html because too much of a hassle to do it
  correctly, and very few people ever do it (mostly me I think).
- Added a bespoke handler / renderer for github-style references.

  Note: uses positional captures because it started that way and named
  captures are not removed from that sequence so mixing and matching
  is not very useful, plus python does not support identically named
  groups (even exclusive) so all 4 repo captures and all 3 issue
  number captures would need different names...
- And added a second bespoke handler for our own opw/issue references
  leading to odoo.com, that's something we can't do via github[^1] so
  it's a genuine value-add.

Fixes #889

[^1]: github can do it (though possibly not with the arbitrary
    unspecified nonsense I got when I tried to list some of the
    reference styles, some folks need therapy), but it's not available
    on our plan
---
 runbot_merge/changelog/2024-08/description.md |  10 +
 runbot_merge/models/pull_requests.py          |  18 +-
 runbot_merge/models/utils.py                  | 191 ++++++++++++++++++
 runbot_merge/tests/test_dfm.py                |  72 +++++++
 runbot_merge/views/templates.xml              |   2 +-
 5 files changed, 291 insertions(+), 2 deletions(-)
 create mode 100644 runbot_merge/changelog/2024-08/description.md
 create mode 100644 runbot_merge/tests/test_dfm.py

diff --git a/runbot_merge/changelog/2024-08/description.md b/runbot_merge/changelog/2024-08/description.md
new file mode 100644
index 00000000..90fdf25b
--- /dev/null
+++ b/runbot_merge/changelog/2024-08/description.md
@@ -0,0 +1,10 @@
+IMP: PR descriptions are now markdown-rendered in the dashboard
+
+Previously the raw text was displayed. The main advantage of rendering, aside
+from not splatting huge links in the middle of the thing, is that we can
+autolink *odoo tasks* if they're of a pattern we recognize. Some support has
+also been added for github's references to mirror GFM rendering.
+
+This would be a lot less useful (and in fact pretty much useless) if we could
+use github's built-in [references to external resources](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/managing-repository-settings/configuring-autolinks-to-reference-external-resources)
+sadly that seems to not be available on our plan.
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 5602f0be..4623cc52 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -16,13 +16,14 @@ from typing import Optional, Union, List, Iterator, Tuple
 import psycopg2
 import sentry_sdk
 import werkzeug
+from markupsafe import Markup
 
 from odoo import api, fields, models, tools, Command
 from odoo.exceptions import AccessError
 from odoo.osv import expression
 from odoo.tools import html_escape, Reverse
 from . import commands
-from .utils import enum, readonly
+from .utils import enum, readonly, dfm
 
 from .. import github, exceptions, controllers, utils
 
@@ -369,6 +370,7 @@ class PullRequests(models.Model):
     )
     refname = fields.Char(compute='_compute_refname')
     message = fields.Text(required=True)
+    message_html = fields.Html(compute='_compute_message_html', sanitize=False)
     draft = fields.Boolean(
         default=False, required=True, tracking=True,
         help="A draft PR can not be merged",
@@ -490,6 +492,20 @@ class PullRequests(models.Model):
         for pr in self:
             pr.message_title = next(iter(pr.message.splitlines()), '')
 
+    @api.depends("message")
+    def _compute_message_html(self):
+        for pr in self:
+            match pr.message.split('\n\n', 1):
+                case [title]:
+                    pr.message_html = Markup('<h3>%s<h3>') % title
+                case [title, description]:
+                    pr.message_html = Markup('<h3>%s</h3>\n%s') % (
+                        title,
+                        dfm(pr.repository.name, description),
+                    )
+                case _:
+                    pr.message_html = ""
+
     @api.depends('repository.name', 'number', 'message')
     def _compute_display_name(self):
         return super()._compute_display_name()
diff --git a/runbot_merge/models/utils.py b/runbot_merge/models/utils.py
index f91eb080..a66fe0b4 100644
--- a/runbot_merge/models/utils.py
+++ b/runbot_merge/models/utils.py
@@ -1,4 +1,11 @@
+import logging
+from contextvars import ContextVar
 from typing import Tuple
+from xml.etree.ElementTree import Element, tostring
+
+import markdown.inlinepatterns
+import markdown.treeprocessors
+from markupsafe import escape, Markup
 
 
 def enum(model: str, field: str) -> Tuple[str, str]:
@@ -8,3 +15,187 @@ def enum(model: str, field: str) -> Tuple[str, str]:
 
 def readonly(_):
     raise TypeError("Field is readonly")
+
+
+DFM_CONTEXT_REPO = ContextVar("dfm_context", default="")
+def dfm(repository: str, text: str) -> Markup:
+    """ Converts the input text from markup to HTML using the Odoo PR
+    Description Rules, which are basically:
+
+    - GFM
+    - minus raw HTML (?)
+    - + github's autolinking (https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/autolinked-references-and-urls)
+    - + bespoke autolinking of OPW and Task links to odoo.com
+    """
+    t = DFM_CONTEXT_REPO.set(repository)
+    try:
+        return Markup(dfm_renderer.convert(escape(text)))
+    finally:
+        DFM_CONTEXT_REPO.reset(t)
+
+
+class DfmExtension(markdown.extensions.Extension):
+    def extendMarkdown(self, md):
+        md.registerExtensions(['fenced_code', 'footnotes', 'nl2br', 'sane_lists', 'tables'], configs={})
+        md.inlinePatterns.register(GithubLinking(md), 'githublinking', 123)
+        md.inlinePatterns.register(OdooLinking(md), 'odoolinking', 124)
+        # ideally the unlinker should run before the prettifier so the
+        # prettification is done correctly, but it seems unlikely the prettifier
+        # handles the variable nature of links correctly, and we likely want to
+        # run after the unescaper
+        md.treeprocessors.register(Unlinker(), "unlinker", -10)
+
+class GithubLinking(markdown.inlinepatterns.InlineProcessor):
+    """Aside from being *very* varied github links are *contextual*. That is,
+    their resolution depends on the repository they're being called from
+    (technically they also need all the information from the github backend to
+    know the people & objects exist but we don't have that option).
+
+    Context is not available to us, but we can fake it through the application
+    of contextvars: ``DFM_CONTEXT_REPO`` should contain the full name of the
+    repository this is being resolved from.
+
+    If ``DFM_CONTEXT_REPO`` is empty and needed, this processor emits a warning.
+    """
+    def __init__(self, md=None):
+        super().__init__(r"""(?xi)
+(?:
+    \bhttps://github.com/([\w\.-]+/[\w\.-]+)/(?:issues|pull)/(\d+)(\#[\w-]+)?
+|   \bhttps://github.com/([\w\.-]+/[\w\.-]+)/commit/([a-f0-9]+)
+|   \b([\w\.-]+/[\w\.-]+)\#(\d+)
+|   (\bGH-|(?:^|(?<=\s))\#)(\d+)
+|   \b(?:
+        # user@sha or user/repo@sha
+        ([\w\.-]+(?:/[\w\.-]+)?)
+        @
+        ([0-9a-f]{7,40})
+    )
+|   \b(
+        # a sha is 7~40 hex digits but that means any million+ number matches
+        # which is probably wrong. So ensure there's at least one letter in the
+        # set by using a positive lookahead which looks for a sequence of at
+        # least 0 numbers followed by a-f
+        (?=[0-9]{0,39}?[a-f])
+        [0-9a-f]{7,40}
+    )
+)
+\b
+""", md)
+
+    def handleMatch(self, m, data):
+        ctx = DFM_CONTEXT_REPO.get()
+        if not ctx:
+            logging.getLogger(__name__)\
+                .getChild("github_links")\
+                .warning("missing context for rewriting github links, skipping")
+            return m[0], *m.span()
+
+        repo = issue = commit = None
+        if m[2]:  # full issue / PR
+            repo = m[1]
+            issue = m[2]
+        elif m[5]:  # long hash
+            repo = m[4]
+            commit = m[5]
+        elif m[7]:  # short issue with repo
+            repo = m[6]
+            issue = m[7]
+        elif m[9]:  # short issue without repo
+            repo = None if m[8] == '#' else "GH"
+            issue = m[9]
+        elif m[11]:  # medium hash
+            repo = m[10]
+            commit = m[11]
+        else:  # hash only
+            commit = m[12]
+
+        el = Element("a")
+        if issue is not None:
+            if repo == "GH":
+                el.text = f"GH-{issue}"
+                repo = ctx
+            elif repo in (None, ctx):
+                repo = ctx
+                el.text = f"#{issue}"
+            else:
+                el.text = f"{repo}#{issue}"
+
+            if (fragment := m[3]) and fragment.startswith('#issuecomment-'):
+                el.text += ' (comment)'
+            else:
+                fragment = ''
+            el.set('href', f"https://github.com/{repo}/issues/{issue}{fragment}")
+        else:
+            if repo in (None, ctx):
+                label_repo = ""
+                repo = ctx
+            elif '/' not in repo:  # owner-only
+                label_repo = repo
+                # NOTE: I assume in reality we're supposed to find the actual fork if unambiguous...
+                repo = repo + '/' + ctx.split('/')[-1]
+            elif repo.split('/')[-1] == ctx.split('/')[-1]:
+                # NOTE: here we assume if it's the same repo in a different owner it's a fork
+                label_repo = repo.split('/')[0]
+            else:
+                label_repo = repo
+            el.text = f"{label_repo}@{commit}" if label_repo else commit
+            el.set("href", f"https://github.com/{repo}/commit/{commit}")
+        return el, *m.span()
+
+
+class OdooLinking(markdown.inlinepatterns.InlineProcessor):
+    def __init__(self, md=None):
+        # there are other weirder variations but fuck em, this matches
+        # "opw", "task", "task-id" or "taskid" followed by an optional - or :
+        # followed by digits
+        super().__init__(r"(?i)\b(task(?:-?id)?|opw)\s*[-:]?\s*(\d+)\b", md)
+
+    def handleMatch(self, m, data):
+        el = Element("a", href='https://www.odoo.com/web#model=project.task&id=' + m[2])
+        if m[1].lower() == 'opw':
+            el.text = f"opw-{m[2]}"
+        else:
+            el.text = f"task-{m[2]}"
+        return el, *m.span()
+
+
+class Unlinker(markdown.treeprocessors.Treeprocessor):
+    def run(self, root):
+        # find all elements which contain a link, as ElementTree does not have
+        # parent links we can't really replace links in place
+        for parent in root.iterfind('.//*[a]'):
+            children = parent[:]
+            # can't use clear because that clears the attributes and tail/text
+            del parent[:]
+            for el in children:
+                if el.tag != 'a' or el.get('href', '').startswith(('https:', 'http:')):
+                    parent.append(el)
+                    continue
+
+                # this is a weird link, remove it
+
+                if el.text:  # first attach its text to the previous element
+                    if len(parent):  # prev is not parent
+                        parent[-1].tail = (parent[-1].tail or '') + el.text
+                    else:
+                        parent.text = (parent.text or '') + el.text
+
+                if len(el):  # then unpack all its children
+                    parent.extend(el[:])
+
+                if el.tail:  # then attach tail to previous element
+                    if len(parent):  # prev is not parent
+                        parent[-1].tail = (parent[-1].tail or '') + el.tail
+                    else:
+                        parent.text = (parent.text or '') + el.tail
+
+        return None
+
+
+# alternatively, use cmarkgfm? The maintainer of py-gfm (impl'd over
+# python-markdown) ultimately gave up, if apparently mostly due to pymarkdown's
+# tendency to break its API all the time
+dfm_renderer = markdown.Markdown(
+    extensions=[DfmExtension()],
+    output_format='html5',
+)
diff --git a/runbot_merge/tests/test_dfm.py b/runbot_merge/tests/test_dfm.py
new file mode 100644
index 00000000..1fac6203
--- /dev/null
+++ b/runbot_merge/tests/test_dfm.py
@@ -0,0 +1,72 @@
+from odoo.addons.runbot_merge.models.utils import dfm
+
+def test_odoo_links():
+    assert dfm("", "OPW-42") == '<p><a href="https://www.odoo.com/web#model=project.task&amp;id=42">opw-42</a></p>'
+    assert dfm("", "taskid : 42") == '<p><a href="https://www.odoo.com/web#model=project.task&amp;id=42">task-42</a></p>'
+    assert dfm("", "I was doing task foo") == '<p>I was doing task foo</p>'
+    assert dfm("", "Task 687d3") == "<p>Task 687d3</p>"
+
+def p(*content):
+    return f'<p>{"".join(content)}</p>'
+def a(label, url):
+    return f'<a href="{url}">{label}</a>'
+def test_gh_issue_links():
+    # same-repository link
+    assert dfm("odoo/runbot", "thing thing #26") == p("thing thing ", a('#26', 'https://github.com/odoo/runbot/issues/26'))
+    assert dfm("odoo/runbot", "GH-26") == p(a('GH-26', 'https://github.com/odoo/runbot/issues/26'))
+    assert dfm(
+        "odoo/runbot", "https://github.com/odoo/runbot/issues/26"
+    ) == p(a('#26', 'https://github.com/odoo/runbot/issues/26'))
+
+    # cross-repo link
+    assert dfm(
+        "odoo/runbot", "jlord/sheetsee.js#26"
+    ) == p(a('jlord/sheetsee.js#26', 'https://github.com/jlord/sheetsee.js/issues/26'))
+    assert dfm(
+        "odoo/runbot", "https://github.com/jlord/sheetsee.js/pull/26"
+    ) == p(a('jlord/sheetsee.js#26', 'https://github.com/jlord/sheetsee.js/issues/26'))
+
+    # cross-repo link with comment
+    assert dfm(
+        "odoo/runbot", "https://github.com/odoo/odoo/pull/173061#issuecomment-2227874482"
+    ) == p(a("odoo/odoo#173061 (comment)", "https://github.com/odoo/odoo/issues/173061#issuecomment-2227874482"))
+
+
+def test_gh_commit_link():
+    # same repository
+    assert dfm(
+        "odoo/runbot", "https://github.com/odoo/runbot/commit/a5c3785ed8d6a35868bc169f07e40e889087fd2e"
+    ) == p(a("a5c3785ed8d6a35868bc169f07e40e889087fd2e", "https://github.com/odoo/runbot/commit/a5c3785ed8d6a35868bc169f07e40e889087fd2e"))
+    # cross fork
+    assert dfm(
+        "odoo/runbot", "jlord@a5c3785ed8d6a35868bc169f07e40e889087fd2e"
+    ) == p(a("jlord@a5c3785ed8d6a35868bc169f07e40e889087fd2e", "https://github.com/jlord/runbot/commit/a5c3785ed8d6a35868bc169f07e40e889087fd2e"))
+    assert dfm(
+        "odoo/runbot", "https://github.com/jlord/runbot/commit/a5c3785ed8d6a35868bc169f07e40e889087fd2e"
+    ) == p(a("jlord@a5c3785ed8d6a35868bc169f07e40e889087fd2e", "https://github.com/jlord/runbot/commit/a5c3785ed8d6a35868bc169f07e40e889087fd2e"))
+    # cross repo
+    assert dfm(
+        "odoo/runbot", "jlord/sheetsee.js@a5c3785ed8d6a35868bc169f07e40e889087fd2e"
+    ) == p(a("jlord/sheetsee.js@a5c3785ed8d6a35868bc169f07e40e889087fd2e", "https://github.com/jlord/sheetsee.js/commit/a5c3785ed8d6a35868bc169f07e40e889087fd2e"))
+    assert dfm(
+        "odoo/runbot", "https://github.com/jlord/sheetsee.js/commit/a5c3785ed8d6a35868bc169f07e40e889087fd2e"
+    ) == p(a("jlord/sheetsee.js@a5c3785ed8d6a35868bc169f07e40e889087fd2e", "https://github.com/jlord/sheetsee.js/commit/a5c3785ed8d6a35868bc169f07e40e889087fd2e"))
+
+def test_standalone_hash():
+    assert dfm(
+        "odoo/runbot", "a5c3785ed8d6a35868bc169f07e40e889087fd2e"
+    ) == p(a("a5c3785ed8d6a35868bc169f07e40e889087fd2e", "https://github.com/odoo/runbot/commit/a5c3785ed8d6a35868bc169f07e40e889087fd2e"))
+    assert dfm(
+        "odoo/runbot", "a5c3785ed8d6a35868bc169f07e4"
+    ) == p(a("a5c3785ed8d6a35868bc169f07e4", "https://github.com/odoo/runbot/commit/a5c3785ed8d6a35868bc169f07e4"))
+    assert dfm(
+        "odoo/runbot", "a5c3785"
+    ) == p(a("a5c3785", "https://github.com/odoo/runbot/commit/a5c3785"))
+    assert dfm(
+        "odoo/runbot", "a5c378"
+    ) == p("a5c378")
+
+def test_ignore_tel():
+    assert dfm("", "[ok](https://github.com)") == p(a("ok", "https://github.com"))
+    assert dfm("", "[nope](tel:+1-212-555-0100)") == "<p>nope</p>"
+    assert dfm("", "[lol](rdar://10198949)") == "<p>lol</p>"
diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index 62b9069c..4b70a56c 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -446,7 +446,7 @@
                     <dd><a t-attf-href="{{pr.github_url}}/commits/{{pr.head}}"><span t-field="pr.head"/></a></dd>
                 </dl>
                 <t t-call="runbot_merge.dashboard-table"/>
-                <p t-field="pr.message"/>
+                <p t-field="pr.message_html"/>
             </div></div>
         </t>
     </template>

From 47df8fac847e6537c8a1b5a3cef8409b22fc70de Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 11 Jul 2024 14:12:18 +0200
Subject: [PATCH 159/192] [FIX] runbot_merge: bg-unmerged color

This color was altered in 232aa271b0f1b8da25ba48c9217837653de39150, it
was moved from a cyan-ish green to a yellow quite close to the warning
color.

There is no explanation why, the commit concerns itself with *PR*
dashboards but this class / color is only used on the main
dashboard. It may have been a victim of the color refactoring in that
commit and I fucked up.

This is very disagreeable as it shows up as basically a warning
between the end of staging and it actually getting merged. Rollback
this change back to a green-cyan.
---
 runbot_merge/static/scss/runbot_merge.scss | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/static/scss/runbot_merge.scss b/runbot_merge/static/scss/runbot_merge.scss
index 92d807a4..ea2a12d4 100644
--- a/runbot_merge/static/scss/runbot_merge.scss
+++ b/runbot_merge/static/scss/runbot_merge.scss
@@ -35,7 +35,7 @@ $mergebot-colors: ("success": #dff0d8, "danger": #f2dede, "warning": #fcf8e3, "i
   }
 }
 .bg-unmerged {
-  background-color: #f8f0e3 !important
+  background-color: #dcefe8 !important
 }
 
 .list-inline {

From e6a743bdc2ff329a79280bb678db72262fe0009f Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 15 Jul 2024 13:49:39 +0200
Subject: [PATCH 160/192] [FIX] runbot_merge: the batch target should
 prioritise open PRs

From the previous version of `_compute_target` this was clearly
intended otherwise the fallback makes no sense, but just as clearly I
completely missed / forgot about it halfway through (and the lack of
test didn't help).

The compute is also way overcomplicated, it's not clear why (the only
explanation I can think of is that an intermediate version had a
string target but if that ever happened it was squashed away).
---
 runbot_merge/models/batch.py                 | 14 ++----
 runbot_merge/tests/test_batch_consistency.py | 52 ++++++++++++++++++--
 2 files changed, 52 insertions(+), 14 deletions(-)

diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index 1d0ec945..2c6e85c2 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -187,19 +187,11 @@ class Batch(models.Model):
     def _search_name(self, operator, value):
         return [('all_prs.label', operator, value)]
 
-    @api.depends("all_prs.target")
+    @api.depends("all_prs.target", "all_prs.closed")
     def _compute_target(self):
         for batch in self:
-            if len(batch.prs) == 1:
-                batch.target = batch.all_prs.target
-            else:
-                targets = set(batch.all_prs.mapped('target'))
-                if not targets:
-                    targets = set(batch.all_prs.mapped('target'))
-                if len(targets) == 1:
-                    batch.target = targets.pop()
-                else:
-                    batch.target = False
+            targets = batch.prs.mapped('target') or batch.all_prs.mapped('target')
+            batch.target = targets if len(targets) == 1 else False
 
     @api.depends(
         "merge_date",
diff --git a/runbot_merge/tests/test_batch_consistency.py b/runbot_merge/tests/test_batch_consistency.py
index 4a35389b..8855bfbb 100644
--- a/runbot_merge/tests/test_batch_consistency.py
+++ b/runbot_merge/tests/test_batch_consistency.py
@@ -33,9 +33,6 @@ def test_close_single(env, repo):
     assert Batches.search_count([]) == 0
 
 def test_close_multiple(env, make_repo2):
-    """If a batch has a single PR and that PR gets closed, the batch should be
-    inactive *and* blocked.
-    """
     Batches = env['runbot_merge.batch']
     repo1 = make_repo2('wheee')
     repo2 = make_repo2('wheeee')
@@ -81,3 +78,52 @@ def test_close_multiple(env, make_repo2):
     assert batch_id.prs == env['runbot_merge.pull_requests'].browse(())
     assert not batch_id.active
     assert Batches.search_count([]) == 0
+
+def test_inconsistent_target(env, project, make_repo2):
+    """If a batch's PRs have inconsistent targets,
+
+    - only open PRs should count
+    - it should be clearly notified on the dash
+    - the dash should not get hopelessly lost
+    - there should be a wizard to split the batch / move a PR to a separate batch
+    """
+    Batches = env['runbot_merge.batch']
+    repo1 = make_repo2('whe')
+    repo2 = make_repo2('whee')
+    repo3 = make_repo2('wheee')
+    project.write({'branch_ids': [(0, 0, {'name': 'other'})]})
+
+    with repo1:
+        repo1.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
+        repo1.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/other')
+        repo1.make_commits('master', Commit('b', tree={"b": "b"}), ref='heads/a_pr')
+        pr1 = repo1.make_pr(head='a_pr', target='master')
+
+    with repo2:
+        repo2.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
+        repo2.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/other')
+        repo2.make_commits('master', Commit('b', tree={"b": "b"}), ref='heads/a_pr')
+        pr2 = repo2.make_pr(head='a_pr', target='master')
+
+    with repo3:
+        repo3.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
+        repo3.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/other')
+        repo3.make_commits('master', Commit('b', tree={"b": "b"}), ref='heads/a_pr')
+        pr3 = repo3.make_pr(head='a_pr', target='master')
+
+    [b] = Batches.search([])
+    assert b.target.name == 'master'
+    assert len(b.prs) == 3
+    assert len(b.all_prs) == 3
+
+    with repo3:
+        pr3.base = 'other'
+    assert b.target.name == False
+    assert len(b.prs) == 3
+    assert len(b.all_prs) == 3
+
+    with repo3:
+        pr3.close()
+    assert b.target.name == 'master'
+    assert len(b.prs) == 2
+    assert len(b.all_prs) == 3

From 4a40c0338ca0d170300218fb20834af4581bfbf2 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 15 Jul 2024 15:14:23 +0200
Subject: [PATCH 161/192] [ADD] runbot_merge: small wizard to split a PR off of
 its batch

---
 runbot_merge/models/pull_requests.py         | 46 +++++++++++++++++++-
 runbot_merge/security/ir.model.access.csv    |  1 +
 runbot_merge/tests/test_batch_consistency.py | 41 ++++++++++++++++-
 runbot_merge/views/mergebot.xml              | 20 ++++++++-
 4 files changed, 103 insertions(+), 5 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 7e8e312a..649332d6 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -19,7 +19,7 @@ import werkzeug
 from markupsafe import Markup
 
 from odoo import api, fields, models, tools, Command
-from odoo.exceptions import AccessError
+from odoo.exceptions import AccessError, UserError
 from odoo.osv import expression
 from odoo.tools import html_escape, Reverse
 from . import commands
@@ -318,6 +318,19 @@ class Branch(models.Model):
             b.active_staging_id = b.with_context(active_test=True).staging_ids
 
 
+class SplitOffWizard(models.TransientModel):
+    _name = "runbot_merge.pull_requests.split_off"
+    _description = "wizard to split a PR off of its current batch and into a different one"
+
+    pr_id = fields.Many2one("runbot_merge.pull_requests", required=True)
+    new_label = fields.Char(string="New Label")
+
+    def button_apply(self):
+        self.pr_id._split_off(self.new_label)
+        self.unlink()
+        return {'type': 'ir.actions.act_window_close'}
+
+
 ACL = collections.namedtuple('ACL', 'is_admin is_reviewer is_author')
 class PullRequests(models.Model):
     _name = 'runbot_merge.pull_requests'
@@ -1605,6 +1618,37 @@ For your own safety I've ignored *everything in your entire comment*.
             format_args=format_args,
         )
 
+    def button_split(self):
+        if len(self.batch_id.prs) == 1:
+            raise UserError("Splitting a batch with a single PR is dumb")
+
+        w = self.env['runbot_merge.pull_requests.split_off'].create({
+            'pr_id': self.id,
+            'new_label': self.label,
+        })
+        return {
+            'type': 'ir.actions.act_window',
+            'res_model': w._name,
+            'res_id': w.id,
+            'target': 'new',
+            'views': [(False, 'form')],
+        }
+
+    def _split_off(self, new_label):
+        # should not be usable to move a PR between batches (maybe later)
+        batch = self.env['runbot_merge.batch']
+        if not re.search(r':patch-\d+$', new_label):
+            if batch.search([
+                ('merge_date', '=', False),
+                ('prs.label', '=', new_label),
+            ]):
+                raise UserError("Can not split off to an existing batch")
+
+        self.write({
+            'label': new_label,
+            'batch_id': batch.create({}).id,
+        })
+
 # ordering is a bit unintuitive because the lowest sequence (and name)
 # is the last link of the fp chain, reasoning is a bit more natural the
 # other way around (highest object is the last), especially with Python
diff --git a/runbot_merge/security/ir.model.access.csv b/runbot_merge/security/ir.model.access.csv
index 9d2da846..76bbf09d 100644
--- a/runbot_merge/security/ir.model.access.csv
+++ b/runbot_merge/security/ir.model.access.csv
@@ -9,6 +9,7 @@ access_runbot_merge_repository_status_admin,Admin access to repo statuses,model_
 access_runbot_merge_branch_admin,Admin access to branches,model_runbot_merge_branch,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_pull_requests_admin,Admin access to PR,model_runbot_merge_pull_requests,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_pull_requests_tagging_admin,Admin access to tagging,model_runbot_merge_pull_requests_tagging,runbot_merge.group_admin,1,1,1,1
+access_runbot_merge_pull_requests_split_admin,Admin access to batch split wizard,model_runbot_merge_pull_requests_split_off,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_commit_admin,Admin access to commits,model_runbot_merge_commit,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_stagings_admin,Admin access to stagings,model_runbot_merge_stagings,runbot_merge.group_admin,1,1,1,1
 access_runbot_merge_stagings_heads_admin,Admin access to staging heads,model_runbot_merge_stagings_heads,runbot_merge.group_admin,1,1,1,1
diff --git a/runbot_merge/tests/test_batch_consistency.py b/runbot_merge/tests/test_batch_consistency.py
index 8855bfbb..18e66d4c 100644
--- a/runbot_merge/tests/test_batch_consistency.py
+++ b/runbot_merge/tests/test_batch_consistency.py
@@ -1,6 +1,8 @@
 """This module tests edge cases specific to the batch objects themselves,
 without wider relevance and thus other location.
 """
+import pytest
+
 from utils import Commit, to_pr
 
 
@@ -79,7 +81,7 @@ def test_close_multiple(env, make_repo2):
     assert not batch_id.active
     assert Batches.search_count([]) == 0
 
-def test_inconsistent_target(env, project, make_repo2):
+def test_inconsistent_target(env, project, make_repo2, users):
     """If a batch's PRs have inconsistent targets,
 
     - only open PRs should count
@@ -87,6 +89,7 @@ def test_inconsistent_target(env, project, make_repo2):
     - the dash should not get hopelessly lost
     - there should be a wizard to split the batch / move a PR to a separate batch
     """
+    # region setup
     Batches = env['runbot_merge.batch']
     repo1 = make_repo2('whe')
     repo2 = make_repo2('whee')
@@ -99,6 +102,9 @@ def test_inconsistent_target(env, project, make_repo2):
         repo1.make_commits('master', Commit('b', tree={"b": "b"}), ref='heads/a_pr')
         pr1 = repo1.make_pr(head='a_pr', target='master')
 
+        repo1.make_commits('master', Commit('b', tree={"c": "c"}), ref='heads/something_else')
+        pr_other = repo1.make_pr(head='something_else', target='master')
+
     with repo2:
         repo2.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
         repo2.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/other')
@@ -111,7 +117,13 @@ def test_inconsistent_target(env, project, make_repo2):
         repo3.make_commits('master', Commit('b', tree={"b": "b"}), ref='heads/a_pr')
         pr3 = repo3.make_pr(head='a_pr', target='master')
 
-    [b] = Batches.search([])
+    assert repo1.owner == repo2.owner == repo3.owner
+    owner = repo1.owner
+    # endregion
+
+    # region closeable consistency
+
+    [b] = Batches.search([('all_prs.label', '=', f'{owner}:a_pr')])
     assert b.target.name == 'master'
     assert len(b.prs) == 3
     assert len(b.all_prs) == 3
@@ -127,3 +139,28 @@ def test_inconsistent_target(env, project, make_repo2):
     assert b.target.name == 'master'
     assert len(b.prs) == 2
     assert len(b.all_prs) == 3
+    # endregion
+
+    # region split batch
+    with repo2:
+        pr2.base = 'other'
+    assert b.target.name == False
+    assert to_pr(env, pr_other).label == f'{owner}:something_else'
+    pr2_id = to_pr(env, pr2)
+    act = pr2_id.button_split()
+    assert act['type'] == 'ir.actions.act_window'
+    assert act['views'] == [[False, 'form']]
+    assert act['target'] == 'new'
+    w = env[act['res_model']].browse([act['res_id']])
+    w.new_label = f"{owner}:something_else"
+    with pytest.raises(Exception):
+        w.button_apply()
+    w.new_label = f"{owner}:blah-blah-blah"
+    w.button_apply()
+
+    assert pr2_id.label == f"{owner}:blah-blah-blah"
+    assert pr2_id.batch_id != to_pr(env, pr1).batch_id
+    assert b.target.name == 'master'
+    assert len(b.prs) == 1, "the PR has been moved off of this batch entirely"
+    assert len(b.all_prs) == 2
+    # endregion
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index eecec149..66888469 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -125,8 +125,10 @@
             <form>
                 <div class="o_form_statusbar">
                     <span class="o_statusbar_buttons">
-                    <field name="github_url" widget="url" class="btn btn-secondary" text="Github"/>
-                    <field name="url" widget="url" class="btn btn-secondary" text="Frontend"/></span>
+                        <button type="object" name="button_split" string="Split Off"/>
+                        <field name="github_url" widget="url" class="btn btn-secondary" text="Github"/>
+                        <field name="url" widget="url" class="btn btn-secondary" text="Frontend"/>
+                    </span>
                 </div>
                 <sheet>
                     <field name="project" invisible="1"/>
@@ -261,6 +263,20 @@
         </field>
     </record>
 
+    <record id="runbot_merge_pull_requests_split_off_form" model="ir.ui.view">
+        <field name="name">Split Off Form</field>
+        <field name="model">runbot_merge.pull_requests.split_off</field>
+        <field name="arch" type="xml">
+            <form>
+                <field name="new_label" colspan="4"/>
+                <footer>
+                    <button type="object" name="button_apply" string="Apply" class="btn btn-primary"/>
+                    <button special="cancel" string="Cancel"  class="btn btn-secondary"/>
+                </footer>
+            </form>
+        </field>
+    </record>
+
     <record id="runbot_merge_action_stagings" model="ir.actions.act_window">
         <field name="name">Stagings</field>
         <field name="res_model">runbot_merge.stagings</field>

From 82ec48c8da8eab105ca7ce65e454e5fbddb5e043 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 15 Jul 2024 15:55:44 +0200
Subject: [PATCH 162/192] [IMP] runbot_merge: track PR label

It's not modified super often, but seems important to track if it
happens to be modified.
---
 runbot_merge/models/pull_requests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 649332d6..054b3025 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -383,7 +383,7 @@ class PullRequests(models.Model):
     author = fields.Many2one('res.partner', index=True)
     head = fields.Char(required=True, tracking=True)
     label = fields.Char(
-        required=True, index=True,
+        required=True, index=True, tracking=True,
         help="Label of the source branch (owner:branchname), used for "
              "cross-repository branch-matching"
     )

From a5c514ad6ee44b26de05519e6011aaa8623189a7 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 17 Jul 2024 13:04:40 +0200
Subject: [PATCH 163/192] [FIX] *bot: copy filestore (shared dir) when copying
 db

Previously we would copy *only* the database itself, which is mostly
not an issue, but when trying to debug *pages* using a browser, that
browser tries to load various assets, which are looked up in the
database, then in the filestore, where they are missing.

This is not an issue in and of itself, but it triggers ugly tracebacks
in the logs which is not convenient, and since
2ab06ca96b6b324823305c2cdf4f4447592926bf it fails the test even if the
test would otherwise work.
---
 conftest.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/conftest.py b/conftest.py
index 99b824e5..8e9d54ef 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import select
+import shutil
 import threading
 from typing import Optional
 
@@ -306,7 +307,8 @@ class DbDict(dict):
                 self[module] = db
                 return db
 
-            d = atexit.enter_context(tempfile.TemporaryDirectory())
+            d = (self._shared_dir / f'shared-{module}')
+            d.mkdir()
             self[module] = db = 'template_%s' % uuid.uuid4()
             subprocess.run([
                 'odoo', '--no-http',
@@ -317,7 +319,7 @@ class DbDict(dict):
                 '--log-level', 'warn'
             ],
                 check=True,
-                env={**os.environ, 'XDG_DATA_HOME': d}
+                env={**os.environ, 'XDG_DATA_HOME': str(d)}
             )
             f.write(db)
             f.flush()
@@ -340,9 +342,18 @@ def dbcache(request, tmp_path_factory, addons_path):
     yield dbs
 
 @pytest.fixture
-def db(request, module, dbcache):
+def db(request, module, dbcache, tmpdir):
+    template_db = dbcache[module]
     rundb = str(uuid.uuid4())
-    subprocess.run(['createdb', '-T', dbcache[module], rundb], check=True)
+    subprocess.run(['createdb', '-T', template_db, rundb], check=True)
+    share = tmpdir.mkdir('share')
+    shutil.copytree(
+        str(dbcache._shared_dir / f'shared-{module}'),
+        str(share),
+        dirs_exist_ok=True,
+    )
+    (share / 'Odoo' / 'filestore' / template_db).rename(
+        share / 'Odoo' / 'filestore' / rundb)
 
     yield rundb
 
@@ -464,7 +475,7 @@ def server(request, db, port, module, addons_path, tmpdir):
         **os.environ,
         # stop putting garbage in the user dirs, and potentially creating conflicts
         # TODO: way to override this with macOS?
-        'XDG_DATA_HOME': str(tmpdir.mkdir('share')),
+        'XDG_DATA_HOME': str(tmpdir / 'share'),
         'XDG_CACHE_HOME': str(tmpdir.mkdir('cache')),
     })
     os.close(w)

From 5126fd8053682b8d7ffcbb13e3177433e49a6aa5 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 17 Jul 2024 15:04:56 +0200
Subject: [PATCH 164/192] [IMP] core: ensure web.base.url is correctly set up

Finally went to fix the issue that web.base.url is not correctly set
up by the test suite and thus things like `PullRequest.url` don't
return the right URL when running tests.

The answer I re-discovered yet again is that one only has to log in as
an admin (`base.group_system`) while providing a `base_location` env
key.

This is done automatically when logging via the frontend, under the
assumption that this is the "correct" url, but when logging in via RPC
it has to be done manually as that's more internal and we might use
e.g. scripts on the same server to manipulate the instance.
---
 conftest.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/conftest.py b/conftest.py
index 8e9d54ef..66afa127 100644
--- a/conftest.py
+++ b/conftest.py
@@ -375,12 +375,14 @@ def wait_for_server(db, port, proc, mod, timeout=120):
 
         try:
             uid = xmlrpc.client.ServerProxy(
-                'http://localhost:{}/xmlrpc/2/common'.format(port))\
-                .authenticate(db, 'admin', 'admin', {})
+                f'http://localhost:{port}/xmlrpc/2/common'
+            ).authenticate(db, 'admin', 'admin', {
+                'base_location': f"http://localhost:{port}",
+            })
             mods = xmlrpc.client.ServerProxy(
-                'http://localhost:{}/xmlrpc/2/object'.format(port))\
-                .execute_kw(
-                    db, uid, 'admin', 'ir.module.module', 'search_read', [
+                f'http://localhost:{port}/xmlrpc/2/object'
+            ).execute_kw(
+                db, uid, 'admin', 'ir.module.module', 'search_read', [
                     [('name', '=', mod)], ['state']
                 ])
             if mods and mods[0].get('state') == 'installed':

From 7a0a6d4415e3d6589960fc681244a71620145f09 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 15 Jul 2024 15:52:08 +0200
Subject: [PATCH 165/192] [IMP] runbot_merge: backend UI

- Update branch name to prefix with project as it can be hard to
  differentiate when filtering by or trying to set targets, given some
  targets are extremely common (e.g. `master`/`main`) and not all
  fields are filtered by project (or even can be).
- Add a proper menu item and list view for batches, maybe it'll be of
  use one day.
- Upgrade label in PR search, it's more likely to be needed than
  author or target.
- Put PRs first in the mergebot menu, as it's *by far* the most likely
  item to look for, unless it's staging in order to cancel one.
---
 runbot_merge/models/batch.py         |  6 ++++-
 runbot_merge/models/pull_requests.py |  8 ++++++-
 runbot_merge/tests/test_basic.py     |  4 ++--
 runbot_merge/views/batch.xml         | 36 ++++++++++++++++++++++++++++
 runbot_merge/views/mergebot.xml      | 19 +++++++--------
 5 files changed, 59 insertions(+), 14 deletions(-)

diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index de70d67b..1d0ec945 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -52,8 +52,9 @@ class Batch(models.Model):
     _description = "batch of pull request"
     _inherit = ['mail.thread']
     _parent_store = True
+    _order = "id desc"
 
-    name = fields.Char(compute="_compute_name")
+    name = fields.Char(compute="_compute_name", search="_search_name")
     target = fields.Many2one('runbot_merge.branch', store=True, compute='_compute_target')
     batch_staging_ids = fields.One2many('runbot_merge.staging.batch', 'runbot_merge_batch_id')
     staging_ids = fields.Many2many(
@@ -183,6 +184,9 @@ class Batch(models.Model):
         for batch in self:
             batch.name = batch.prs[:1].label or batch.all_prs[:1].label
 
+    def _search_name(self, operator, value):
+        return [('all_prs.label', operator, value)]
+
     @api.depends("all_prs.target")
     def _compute_target(self):
         for batch in self:
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 4623cc52..7e8e312a 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -284,7 +284,13 @@ class Branch(models.Model):
             self._table, ['name', 'project_id'])
         return res
 
-    @api.depends('active')
+    def name_get(self):
+        return [
+            (id, f"{b.project_id.name}:{name}")
+            for b, (id, name) in zip(self, super().name_get())
+        ]
+
+    @api.depends('active', 'project_id.name')
     def _compute_display_name(self):
         super()._compute_display_name()
         for b in self.filtered(lambda b: not b.active):
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 3b982984..c4be7fc5 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -770,7 +770,7 @@ def test_ff_failure_batch(env, repo, users, config):
     }
 
 class TestPREdition:
-    def test_edit(self, env, repo, config):
+    def test_edit(self, env, project, repo, config):
         """ Editing PR:
 
         * title (-> message)
@@ -814,7 +814,7 @@ class TestPREdition:
         assert pr.target == branch_1
         assert not pr.staging_id, "updated the base of a staged PR should have unstaged it"
         assert st.state == 'cancelled', f"expected cancellation, got {st.state}"
-        assert st.reason == f"{pr.display_name} target (base) branch was changed from 'master' to '1.0'"
+        assert st.reason == f"{pr.display_name} target (base) branch was changed from '{project.name}:master' to '{project.name}:1.0'"
 
         with repo: prx.base = '2.0'
         assert not pr.exists()
diff --git a/runbot_merge/views/batch.xml b/runbot_merge/views/batch.xml
index 7d87d94f..ed3f04ba 100644
--- a/runbot_merge/views/batch.xml
+++ b/runbot_merge/views/batch.xml
@@ -1,4 +1,40 @@
 <odoo>
+    <record id="runbot_merge_action_batches" model="ir.actions.act_window">
+        <field name="name">Batches</field>
+        <field name="res_model">runbot_merge.batch</field>
+        <field name="view_mode">tree,form</field>
+    </record>
+
+    <record id="runbot_merge_batch_search" model="ir.ui.view">
+        <field name="name">batches search</field>
+        <field name="model">runbot_merge.batch</field>
+        <field name="arch" type="xml">
+            <search>
+                <filter name="all" domain="['|', ('active', '=', True), ('active', '=', False)]"/>
+                <filter name="inactive" domain="[('active', '=', False)]"/>
+
+                <field name="name"/>
+                <field name="target"/>
+                <field name="id"/>
+            </search>
+        </field>
+    </record>
+
+    <record id="runbot_merge_batch_tree" model="ir.ui.view">
+        <field name="name">batches list</field>
+        <field name="model">runbot_merge.batch</field>
+        <field name="arch" type="xml">
+            <tree decoration-muted="not active">
+                <field name="id"/>
+                <field name="name"/>
+                <field name="target"/>
+                <field name="prs" widget="many2many_tags"/>
+                <field name="blocked"/>
+                <field name="active" invisible="1"/>
+            </tree>
+        </field>
+    </record>
+
     <record id="runbot_merge_batch_form" model="ir.ui.view">
         <field name="name">Batch form</field>
         <field name="model">runbot_merge.batch</field>
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index 3de9738e..eecec149 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -88,9 +88,9 @@
                     name="open" string="Open"
                     domain="[('state', 'not in', ['merged', 'closed'])]"
                 />
+                <field name="label"/>
                 <field name="number"/>
                 <field name="author"/>
-                <field name="label"/>
                 <field name="target"/>
                 <field name="repository"/>
                 <field name="state"/>
@@ -416,17 +416,16 @@
         </field>
     </record>
 
-    <menuitem name="Mergebot" id="runbot_merge_menu"/>
-        <menuitem name="Projects" id="runbot_merge_menu_project"
-                  parent="runbot_merge_menu"
-                  action="runbot_merge_action_projects"/>
+    <menuitem name="Mergebot" id="runbot_merge_menu">
         <menuitem name="Pull Requests" id="runbot_merge_menu_prs"
-                  parent="runbot_merge_menu"
-                  action="runbot_merge_action_prs"/>
+                  action="runbot_merge_action_prs" sequence="5"/>
         <menuitem name="Stagings" id="runbot_merge_menu_stagings"
-                  parent="runbot_merge_menu"
-                  action="runbot_merge_action_stagings"/>
+                  action="runbot_merge_action_stagings" sequence="8"/>
+        <menuitem name="Projects" id="runbot_merge_menu_project"
+                  action="runbot_merge_action_projects"/>
+        <menuitem name="Batches" id="runbot_merge_menu_batches"
+                  action="runbot_merge_action_batches"/>
         <menuitem name="Commits" id="runbot_merge_menu_commits"
-                  parent="runbot_merge_menu"
                   action="runbot_merge_action_commits"/>
+    </menuitem>
 </odoo>

From 015c97b2cc6b77e6714aa24b21caeee27cb60865 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 18 Jul 2024 12:13:52 +0200
Subject: [PATCH 166/192] [IMP] runbot_merge: add an alert that a batch is
 inconsistent

Trying to fit an inconsistent batch in the nice table turns out to be
quite difficult (for me anyway) so give up and display *just* the
alert.
---
 runbot_merge/tests/test_batch_consistency.py | 20 +++++++++++++++++---
 runbot_merge/views/templates.xml             | 14 ++++++++++----
 2 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/runbot_merge/tests/test_batch_consistency.py b/runbot_merge/tests/test_batch_consistency.py
index 18e66d4c..e99e1fc2 100644
--- a/runbot_merge/tests/test_batch_consistency.py
+++ b/runbot_merge/tests/test_batch_consistency.py
@@ -3,7 +3,7 @@ without wider relevance and thus other location.
 """
 import pytest
 
-from utils import Commit, to_pr
+from utils import Commit, to_pr, pr_page
 
 
 def test_close_single(env, repo):
@@ -81,7 +81,7 @@ def test_close_multiple(env, make_repo2):
     assert not batch_id.active
     assert Batches.search_count([]) == 0
 
-def test_inconsistent_target(env, project, make_repo2, users):
+def test_inconsistent_target(env, project, make_repo2, users, page):
     """If a batch's PRs have inconsistent targets,
 
     - only open PRs should count
@@ -142,11 +142,25 @@ def test_inconsistent_target(env, project, make_repo2, users):
     # endregion
 
     # region split batch
+    pr1_id = to_pr(env, pr1)
+    pr2_id = to_pr(env, pr2)
     with repo2:
         pr2.base = 'other'
+
+    pr2_dashboard = pr_page(page, pr2)
+    # The dashboard should have an alert
+    s = pr2_dashboard.cssselect('.alert.alert-danger')
+    assert s, "the dashboard should have an alert"
+    assert s[0].text_content().strip() == f"""\
+Inconsistent targets:
+
+{pr1_id.display_name} has target 'master'
+{pr2_id.display_name} has target 'other'\
+"""
+    assert not pr2_dashboard.cssselect('table'), "the batches table should be suppressed"
+
     assert b.target.name == False
     assert to_pr(env, pr_other).label == f'{owner}:something_else'
-    pr2_id = to_pr(env, pr2)
     act = pr2_id.button_split()
     assert act['type'] == 'ir.actions.act_window'
     assert act['views'] == [[False, 'form']]
diff --git a/runbot_merge/views/templates.xml b/runbot_merge/views/templates.xml
index 4b70a56c..2dff930f 100644
--- a/runbot_merge/views/templates.xml
+++ b/runbot_merge/views/templates.xml
@@ -488,10 +488,11 @@ action = (project, repos, branches, genealogy)
         <field name="model_id" ref="base.model_ir_qweb"/>
         <field name="code"><![CDATA[
 batches = {}
-for branch in branches:
-    # FIXME: batches with inconsistent targets?
+for branch in [*branches, branches.browse(())]:
     if genealogy:
         prs_batch = genealogy.filtered(lambda b: b.target == branch).all_prs
+        if not (branch or prs_batch):
+            continue
     else:
         prs_batch = pr
     for repo in repos:
@@ -546,7 +547,12 @@ action = batches
             'branches': branches,
             'genealogy': pre[3],
         })"/>
-        <table class="table table-bordered table-sm">
+        <div t-if="not pr.batch_id.target" class="alert alert-danger">
+<p>Inconsistent targets:</p>
+<ul><li t-foreach="pr.batch_id.prs" t-as="p">
+<a t-att-href="p.url"><t t-out="p.display_name"/></a> has target '<t t-out="p.target.name"/>'</li></ul>
+        </div>
+        <table t-else="" class="table table-bordered table-sm">
             <colgroup>
                 <col/>
                 <col t-foreach="repos" t-as="repo"
@@ -573,7 +579,7 @@ action = batches
                     }} {{
                         'inactive' if not branch.active else ''
                     }}">
-                    <td t-out="branch.name"/>
+                    <td t-out="branch.name or ''"/>
                     <t t-foreach="repos" t-as="repo">
                         <t t-set="ps" t-value="batches[repo, branch]"/>
                         <t t-set="stateclass" t-value="ps['state'] and 'table-'+ps['state']"/>

From fd6eae0d1d24123824b166461e2e65ed250cc473 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 18 Jul 2024 12:34:42 +0200
Subject: [PATCH 167/192] [IMP] runbot_merge: ensure inconsistent batches don't
 merge

I was pretty sure it wouldn't happen but it doesn't hurt to make sure,
also to check that splitting the batch does correctly make things work
out.
---
 runbot_merge/tests/test_batch_consistency.py | 27 +++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/runbot_merge/tests/test_batch_consistency.py b/runbot_merge/tests/test_batch_consistency.py
index e99e1fc2..9298f143 100644
--- a/runbot_merge/tests/test_batch_consistency.py
+++ b/runbot_merge/tests/test_batch_consistency.py
@@ -81,7 +81,7 @@ def test_close_multiple(env, make_repo2):
     assert not batch_id.active
     assert Batches.search_count([]) == 0
 
-def test_inconsistent_target(env, project, make_repo2, users, page):
+def test_inconsistent_target(env, project, make_repo2, users, page, config):
     """If a batch's PRs have inconsistent targets,
 
     - only open PRs should count
@@ -97,8 +97,8 @@ def test_inconsistent_target(env, project, make_repo2, users, page):
     project.write({'branch_ids': [(0, 0, {'name': 'other'})]})
 
     with repo1:
-        repo1.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
-        repo1.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/other')
+        [m] = repo1.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
+        repo1.make_ref('heads/other', m)
         repo1.make_commits('master', Commit('b', tree={"b": "b"}), ref='heads/a_pr')
         pr1 = repo1.make_pr(head='a_pr', target='master')
 
@@ -161,6 +161,19 @@ Inconsistent targets:
 
     assert b.target.name == False
     assert to_pr(env, pr_other).label == f'{owner}:something_else'
+    # try staging
+    with repo1:
+        pr1.post_comment("hansen r+", config['role_reviewer']['token'])
+        repo1.post_status(pr1.head, "success")
+    with repo2:
+        pr2.post_comment("hansen r+", config['role_reviewer']['token'])
+        repo2.post_status(pr2.head, "success")
+    env.run_crons()
+    assert not pr1_id.blocked
+    assert not pr2_id.blocked
+    assert b.blocked
+    assert env['runbot_merge.stagings'].search_count([]) == 0
+
     act = pr2_id.button_split()
     assert act['type'] == 'ir.actions.act_window'
     assert act['views'] == [[False, 'form']]
@@ -178,3 +191,11 @@ Inconsistent targets:
     assert len(b.prs) == 1, "the PR has been moved off of this batch entirely"
     assert len(b.all_prs) == 2
     # endregion
+
+    assert not pr1_id.blocked
+    assert not pr1_id.batch_id.blocked
+    assert not pr2_id.blocked
+    assert not pr2_id.batch_id.blocked
+    env.run_crons()
+
+    assert env['runbot_merge.stagings'].search_count([])

From bca8adbdc46dab53f409fe8e689c544a20c4dfc8 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 24 Jul 2024 12:35:21 +0200
Subject: [PATCH 168/192] [IMP] runbot_merge: move inconsistency block higher
 in batches

Should probably take priority over a PR being misconfigured.
---
 runbot_merge/models/batch.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index 2c6e85c2..eace55b4 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -95,7 +95,7 @@ class Batch(models.Model):
         column_type=enum(_name, 'priority'),
     )
 
-    blocked = fields.Char(store=True, compute="_compute_stageable")
+    blocked = fields.Char(store=True, compute="_compute_blocked")
 
     # unlike on PRs, this does not get detached... ? (because batches can be
     # partially detached so that's a PR-level concern)
@@ -199,12 +199,14 @@ class Batch(models.Model):
         "skipchecks",
         "prs.status", "prs.reviewed_by", "prs.target",
     )
-    def _compute_stageable(self):
+    def _compute_blocked(self):
         for batch in self:
             if batch.merge_date:
                 batch.blocked = "Merged."
             elif not batch.active:
                 batch.blocked = "all prs are closed"
+            elif len(targets := batch.prs.mapped('target')) > 1:
+                batch.blocked = f"Multiple target branches: {', '.join(targets.mapped('name'))!r}"
             elif blocking := batch.prs.filtered(
                 lambda p: p.error or p.draft or not (p.squash or p.merge_method)
             ):
@@ -220,8 +222,6 @@ class Batch(models.Model):
                     unvalidated and f"{unvalidated} are waiting for CI",
                     failed and f"{failed} have failed CI",
                 ]))
-            elif len(targets := batch.prs.mapped('target')) > 1:
-                batch.blocked = f"Multiple target branches: {', '.join(targets.mapped('name'))!r}"
             else:
                 if batch.blocked and batch.cancel_staging:
                     if splits := batch.target.split_ids:

From 6cc9a6ca11ebe1bf6c038a985257f1ee24df7d08 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 24 Jul 2024 12:36:05 +0200
Subject: [PATCH 169/192] [IMP] runbot_merge: show batch inconsistency in PR
 dash picture

Extract current table generation into a separate function, add an
other function to render an alert / list of PR targets if the batch is
not consistent.

This means an extra pass on the table contents to precompute the image
size, but we can delay loading fonts until after etag computation
which might be a bigger gain all things considered: there aren't many
cells in most PR tables, but fonts are rather expensive to
load (I should probably load them at import and cache them in the module...)
---
 runbot_merge/controllers/dashboard.py | 120 +++++++++++++++++---------
 1 file changed, 80 insertions(+), 40 deletions(-)

diff --git a/runbot_merge/controllers/dashboard.py b/runbot_merge/controllers/dashboard.py
index 6a5dd2de..7c5f1f66 100644
--- a/runbot_merge/controllers/dashboard.py
+++ b/runbot_merge/controllers/dashboard.py
@@ -170,13 +170,6 @@ def raster_render(pr):
     if request.httprequest.headers.get('If-Modified-Since') == last_modified:
         return werkzeug.wrappers.Response(status=304, headers=headers)
 
-    with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Regular.ttf', 'rb') as f:
-        font = ImageFont.truetype(f, size=16, layout_engine=0)
-        f.seek(0)
-        supfont = ImageFont.truetype(f, size=13, layout_engine=0)
-    with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Bold.ttf', 'rb') as f:
-        bold = ImageFont.truetype(f, size=16, layout_engine=0)
-
     batches = pr.env.ref('runbot_merge.dashboard-prep')._run_action_code_multi({
         'pr': pr,
         'repos': repos,
@@ -184,31 +177,48 @@ def raster_render(pr):
         'genealogy': genealogy,
     })
 
-    # getbbox returns (left, top, right, bottom)
-
-    rows = {b: font.getbbox(b.name)[3] for b in branches}
-    rows[None] = max(bold.getbbox(r.name)[3] for r in repos)
-
-    columns = {r: bold.getbbox(r.name)[2] for r in repos}
-    columns[None] = max(font.getbbox(b.name)[2] for b in branches)
-
-    etag = hashlib.sha256(f"(P){pr.id},{pr.repository.id},{pr.target.id}".encode())
+    etag = hashlib.sha256(f"(P){pr.id},{pr.repository.id},{pr.target.id},{pr.batch_id.blocked}".encode())
     # repos and branches should be in a consistent order so can just hash that
     etag.update(''.join(f'(R){r.name}' for r in repos).encode())
     etag.update(''.join(f'(T){b.name},{b.active}' for b in branches).encode())
     # and product of deterministic iterations should be deterministic
     for r, b in product(repos, branches):
         ps = batches[r, b]
-        etag.update(f"(B){ps['state']},{ps['detached']},{ps['active']}".encode())
-        # technically label (state + blocked) does not actually impact image
-        # render (though subcomponents of state do) however blocked is useful
-        # to force an etag miss so keeping it
 
+        etag.update(f"(B){ps['state']},{ps['detached']},{ps['active']}".encode())
         etag.update(''.join(
             f"(PS){p['label']},{p['closed']},{p['number']},{p['checked']},{p['reviewed']},{p['attached']},{p['pr'].staging_id.id}"
             for p in ps['prs']
         ).encode())
 
+    etag = headers['ETag'] = base64.b32encode(etag.digest()).decode()
+    if if_none_match == etag:
+        return werkzeug.wrappers.Response(status=304, headers=headers)
+
+    if not pr.batch_id.target:
+        im = render_inconsistent_batch(pr.batch_id)
+    else:
+        im = render_full_table(pr, branches, repos, batches)
+
+    buffer = io.BytesIO()
+    im.save(buffer, 'png', optimize=True)
+    return werkzeug.wrappers.Response(buffer.getvalue(), headers=headers)
+
+
+def render_full_table(pr, branches, repos, batches):
+    with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Regular.ttf', 'rb') as f:
+        font = ImageFont.truetype(f, size=16, layout_engine=0)
+        f.seek(0)
+        supfont = ImageFont.truetype(f, size=13, layout_engine=0)
+    with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Bold.ttf', 'rb') as f:
+        bold = ImageFont.truetype(f, size=16, layout_engine=0)
+    # getbbox returns (left, top, right, bottom)
+    rows = {b: font.getbbox(b.name)[3] for b in branches}
+    rows[None] = max(bold.getbbox(r.name)[3] for r in repos)
+    columns = {r: bold.getbbox(r.name)[2] for r in repos}
+    columns[None] = max(font.getbbox(b.name)[2] for b in branches)
+    for r, b in product(repos, branches):
+        ps = batches[r, b]
         w = h = 0
         for p in ps['prs']:
             _, _, ww, hh = font.getbbox(f" #{p['number']}")
@@ -223,17 +233,12 @@ def raster_render(pr):
         rows[b] = max(rows.get(b, 0), h)
         columns[r] = max(columns.get(r, 0), w)
 
-    etag = headers['ETag'] = base64.b32encode(etag.digest()).decode()
-    if if_none_match == etag:
-        return werkzeug.wrappers.Response(status=304, headers=headers)
-
     pad_w, pad_h = 20, 5
     image_height = sum(rows.values()) + 2 * pad_h * len(rows)
     image_width = sum(columns.values()) + 2 * pad_w * len(columns)
     im = Image.new("RGB", (image_width+1, image_height+1), color='white')
     draw = ImageDraw.Draw(im, 'RGB')
     draw.font = font
-
     # for reasons of that being more convenient we store the bottom of the
     # current row, so getting the top edge requires subtracting h
     w = left = bottom = 0
@@ -244,12 +249,12 @@ def raster_render(pr):
         background = BG['info'] if b == pr.target or r == pr.repository else BG[None]
         w, h = columns[r] + 2 * pad_w, rows[b] + 2 * pad_h
 
-        if r is None: # branch cell in row
+        if r is None:  # branch cell in row
             left = 0
             bottom += h
             if b:
                 draw.rectangle(
-                    (left + 1, bottom - h + 1, left+w - 1, bottom - 1),
+                    (left + 1, bottom - h + 1, left + w - 1, bottom - 1),
                     background,
                 )
                 draw.text(
@@ -257,14 +262,14 @@ def raster_render(pr):
                     b.name,
                     fill=blend(TEXT, opacity, over=background),
                 )
-        elif b is None: # repo cell in top row
-            draw.rectangle((left + 1, bottom - h + 1, left+w - 1, bottom - 1), background)
+        elif b is None:  # repo cell in top row
+            draw.rectangle((left + 1, bottom - h + 1, left + w - 1, bottom - 1), background)
             draw.text((left + pad_w, bottom - h + pad_h), r.name, fill=TEXT, font=bold)
         # draw the bottom-right edges of the cell
         draw.line([
-            (left, bottom), # bottom-left
-            (left + w, bottom), # bottom-right
-            (left+w, bottom-h) # top-right
+            (left, bottom),  # bottom-left
+            (left + w, bottom),  # bottom-right
+            (left + w, bottom - h)  # top-right
         ], fill=(172, 176, 170))
         if r is None or b is None:
             continue
@@ -275,7 +280,7 @@ def raster_render(pr):
         if pr in ps['pr_ids']:
             bgcolor = lighten(bgcolor, by=-0.05)
         background = blend(bgcolor, opacity, over=background)
-        draw.rectangle((left + 1, bottom - h + 1, left+w - 1, bottom - 1), background)
+        draw.rectangle((left + 1, bottom - h + 1, left + w - 1, bottom - 1), background)
 
         top = bottom - h + pad_h
         offset = left + pad_w
@@ -286,13 +291,13 @@ def raster_render(pr):
             x, _, ww, hh = font.getbbox(label)
             if p['closed']:
                 draw.line([
-                    (offset+x, top + hh - hh/3),
-                    (offset+x+ww, top + hh - hh/3),
+                    (offset + x, top + hh - hh / 3),
+                    (offset + x + ww, top + hh - hh / 3),
                 ], fill=foreground)
             offset += ww
             if not p['attached']:
                 # overdraw top border to mark the detachment
-                draw.line([(left, bottom-h), (left+w, bottom-h)], fill=ERROR)
+                draw.line([(left, bottom - h), (left + w, bottom - h)], fill=ERROR)
             for attribute in filter(None, [
                 'error' if p['pr'].error else '',
                 '' if p['checked'] else 'missing statuses',
@@ -300,17 +305,43 @@ def raster_render(pr):
                 '' if p['attached'] else 'detached',
                 'staged' if p['pr'].staging_id else 'ready' if p['pr']._ready else ''
             ]):
-                label = f' {attribute}'
                 color = SUCCESS if attribute in ('staged', 'ready') else ERROR
+                label = f' {attribute}'
                 draw.text((offset, top), label,
                           fill=blend(color, opacity, over=background),
                           font=supfont)
                 offset += supfont.getbbox(label)[2]
             offset += math.ceil(supfont.getlength(" "))
 
-    buffer = io.BytesIO()
-    im.save(buffer, 'png', optimize=True)
-    return werkzeug.wrappers.Response(buffer.getvalue(), headers=headers)
+    return im
+
+
+def render_inconsistent_batch(batch):
+    """If a batch has inconsistent targets, just point out the inconsistency by
+    listing the PR and targets
+    """
+    with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Regular.ttf', 'rb') as f:
+        font = ImageFont.truetype(f, size=16, layout_engine=0)
+
+    im = Image.new("RGB", (4000, 4000), color=BG['danger'])
+    w = h = 0
+    def draw(label, draw=ImageDraw.Draw(im)):
+        nonlocal w, h
+
+        draw.text((0, h), label, fill=blend(ERROR, 1.0, over=BG['danger']), font=font)
+
+        _, _, ww, hh = font.getbbox(label)
+        w = max(w, ww)
+        h += hh
+
+    draw(" Inconsistent targets:")
+    for p in batch.prs:
+        draw(f" • {p.display_name} has target '{p.target.name}'")
+    draw(" To resolve, either retarget or close the mis-targeted pull request(s).")
+
+    return im.crop((0, 0, w+10, h+5))
+
+
 
 Color = Tuple[int, int, int]
 TEXT: Color = (102, 102, 102)
@@ -322,6 +353,15 @@ BG: Mapping[str | None, Color] = collections.defaultdict(lambda: (255, 255, 255)
     'warning': (252, 248, 227),
     'danger': (242, 222, 222),
 })
+
+
+CHECK_MARK = "\uf00c"
+CROSS = "\uf00d"
+BOX_EMPTY = "\uf096"
+DARK_BOX = "\uf0c8"
+DARK_CHECK = "\uf14a"
+
+
 def blend_single(c: int, over: int, opacity: float) -> int:
     return round(over * (1 - opacity) + c * opacity)
 

From f5eb7447fbef5e195f504c3e35c4099ba5f85fdc Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 25 Jul 2024 15:27:35 +0200
Subject: [PATCH 170/192] [IMP] runbot_merge: reorganise composition of PR
 dashboard pic

The previous version worked but was extremely plodding and
procedural. Initially I wanted to compose the table in a single pass
but that turns out not to really be possible as the goal for #908 is
to have a "drawer" for extended information about the current batch:
this means different cells of the same row can have different heights,
so we can't one-pass the image either vertically (later cells of the
current column might be wider) or horizontally (later cells of the
current row might be taller).

However what can be done is give the entire thing *structure*,
essentially defining a very cut down and ad-hoc layout system before
committing the layout to raster.

This also deduplicates formatting and labelling information which was
previously in both the computation first step and the rasterisation
second step.
---
 runbot_merge/controllers/dashboard.py | 256 ++++++++++++++++----------
 1 file changed, 158 insertions(+), 98 deletions(-)

diff --git a/runbot_merge/controllers/dashboard.py b/runbot_merge/controllers/dashboard.py
index 7c5f1f66..7767ba97 100644
--- a/runbot_merge/controllers/dashboard.py
+++ b/runbot_merge/controllers/dashboard.py
@@ -8,11 +8,14 @@ import hashlib
 import io
 import json
 import logging
-import math
 import pathlib
+from dataclasses import dataclass
 from email.utils import formatdate
+from enum import Flag, auto
+from functools import cached_property
 from itertools import chain, product
-from typing import Tuple, cast, Mapping
+from math import ceil
+from typing import Tuple, cast, Mapping, Optional, List
 
 import markdown
 import markupsafe
@@ -23,6 +26,8 @@ from PIL import Image, ImageDraw, ImageFont
 from odoo.http import Controller, route, request
 from odoo.tools import file_open
 
+HORIZONTAL_PADDING = 20
+VERTICAL_PADDING = 5
 
 _logger = logging.getLogger(__name__)
 
@@ -204,6 +209,83 @@ def raster_render(pr):
     im.save(buffer, 'png', optimize=True)
     return werkzeug.wrappers.Response(buffer.getvalue(), headers=headers)
 
+class Decoration(Flag):
+    STRIKETHROUGH = auto()
+
+@dataclass(frozen=True)
+class Text:
+    content: str
+    font: ImageFont.FreeTypeFont
+    color: Color
+    decoration: Decoration = Decoration(0)
+
+    @cached_property
+    def width(self) -> int:
+        return ceil(self.font.getlength(self.content))
+
+    @property
+    def height(self) -> int:
+        return sum(self.font.getmetrics())
+
+    def draw(self, image: ImageDraw.ImageDraw, left: int, top: int):
+        image.text((left, top), self.content, fill=self.color, font=self.font)
+        if Decoration.STRIKETHROUGH in self.decoration:
+            x1, _, x2, _ = self.font.getbbox(self.content)
+            _, y1, _, y2 = self.font.getbbox('x')
+            # put the strikethrough line about 1/3rd down the x (default seems
+            # to be a bit above halfway down but that's ugly with numbers which
+            # is most of our stuff)
+            y = top + y1 + (y2 - y1) / 3
+            image.line([(left + x1, y), (left + x2, y)], self.color)
+
+@dataclass(frozen=True)
+class Line:
+    spans: List[Text]
+
+    @property
+    def width(self) -> int:
+        return sum(s.width for s in self.spans)
+
+    @property
+    def height(self) -> int:
+        return max(s.height for s in self.spans) if self.spans else 0
+
+    def draw(self, image: ImageDraw.ImageDraw, left: int, top: int):
+        for span in self.spans:
+            span.draw(image, left, top)
+            left += span.width
+
+@dataclass(frozen=True)
+class Lines:
+    lines: List[Line]
+
+    @property
+    def width(self) -> int:
+        return max(l.width for l in self.lines)
+
+    @property
+    def height(self) -> int:
+        return sum(l.height for l in self.lines)
+
+    def draw(self, image: ImageDraw.ImageDraw, left: int, top: int):
+        for line in self.lines:
+            line.draw(image, left, top)
+            top += line.height
+
+@dataclass(frozen=True)
+class Cell:
+    content: Lines | Line | Text
+    background: Color = (255, 255, 255)
+    attached: bool = True
+
+    @cached_property
+    def width(self) -> int:
+        return self.content.width + 2 * HORIZONTAL_PADDING
+
+    @cached_property
+    def height(self) -> int:
+        return self.content.height + 2 * VERTICAL_PADDING
+
 
 def render_full_table(pr, branches, repos, batches):
     with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Regular.ttf', 'rb') as f:
@@ -212,106 +294,84 @@ def render_full_table(pr, branches, repos, batches):
         supfont = ImageFont.truetype(f, size=13, layout_engine=0)
     with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Bold.ttf', 'rb') as f:
         bold = ImageFont.truetype(f, size=16, layout_engine=0)
-    # getbbox returns (left, top, right, bottom)
-    rows = {b: font.getbbox(b.name)[3] for b in branches}
-    rows[None] = max(bold.getbbox(r.name)[3] for r in repos)
-    columns = {r: bold.getbbox(r.name)[2] for r in repos}
-    columns[None] = max(font.getbbox(b.name)[2] for b in branches)
-    for r, b in product(repos, branches):
-        ps = batches[r, b]
-        w = h = 0
-        for p in ps['prs']:
-            _, _, ww, hh = font.getbbox(f" #{p['number']}")
-            w += ww + supfont.getbbox(' '.join(filter(None, [
-                'error' if p['pr'].error else '',
-                '' if p['checked'] else 'missing statuses',
-                '' if p['reviewed'] else 'missing r+',
-                '' if p['attached'] else 'detached',
-                'staged' if p['pr'].staging_id else 'ready' if p['pr']._ready else '',
-            ])))[2]
-            h = max(hh, h)
-        rows[b] = max(rows.get(b, 0), h)
-        columns[r] = max(columns.get(r, 0), w)
 
-    pad_w, pad_h = 20, 5
-    image_height = sum(rows.values()) + 2 * pad_h * len(rows)
-    image_width = sum(columns.values()) + 2 * pad_w * len(columns)
-    im = Image.new("RGB", (image_width+1, image_height+1), color='white')
+    rowheights = collections.defaultdict(int)
+    colwidths = collections.defaultdict(int)
+    cells = {}
+    for b in chain([None], branches):
+        for r in chain([None], repos):
+            opacity = 1.0 if b is None or b.active else 0.5
+            current_row = b == pr.target
+            background = BG['info'] if current_row or r == pr.repository else BG[None]
+
+            if b is None: # first row
+                cell = Cell(Text("" if r is None else r.name, bold, TEXT), background)
+            elif r is None: # first column
+                cell = Cell(Text(b.name, font, blend(TEXT, opacity, over=background)), background)
+            else:
+                ps = batches[r, b]
+                bgcolor = lighten(BG[ps['state']], by=-0.05) if pr in ps['pr_ids'] else BG[ps['state']]
+                background = blend(bgcolor, opacity, over=background)
+                foreground = blend((39, 110, 114), opacity, over=background)
+
+                line = []
+                attached = True
+                for p in ps['prs']:
+                    line.append(Text(
+                        f"#{p['number']}",
+                        font,
+                        foreground,
+                        decoration=Decoration.STRIKETHROUGH if p['closed'] else Decoration(0),
+                    ))
+                    attached = attached and p['attached']
+                    for attribute in filter(None, [
+                        'error' if p['pr'].error else '',
+                        '' if p['checked'] else 'missing statuses',
+                        '' if p['reviewed'] else 'missing r+',
+                        '' if p['attached'] else 'detached',
+                        'staged' if p['pr'].staging_id else 'ready' if p['pr']._ready else ''
+                    ]):
+                        color = SUCCESS if attribute in ('staged', 'ready') else ERROR
+                        line.append(Text(f' {attribute}', supfont, blend(color, opacity, over=background)))
+                    line.append(Text(" ", font, foreground))
+                cell = Cell(Line(line), background, attached)
+
+            cells[r, b] = cell
+            rowheights[b] = max(rowheights[b], cell.height)
+            colwidths[r] = max(colwidths[r], cell.width)
+
+    im = Image.new("RGB", (sum(colwidths.values()), sum(rowheights.values())), "white")
+    # no need to set the font here because every text element has its own
     draw = ImageDraw.Draw(im, 'RGB')
-    draw.font = font
-    # for reasons of that being more convenient we store the bottom of the
-    # current row, so getting the top edge requires subtracting h
-    w = left = bottom = 0
-    for b, r in product(chain([None], branches), chain([None], repos)):
-        left += w
+    top = 0
+    for b in chain([None], branches):
+        left = 0
+        for r in chain([None], repos):
+            cell = cells[r, b]
 
-        opacity = 1.0 if b is None or b.active else 0.5
-        background = BG['info'] if b == pr.target or r == pr.repository else BG[None]
-        w, h = columns[r] + 2 * pad_w, rows[b] + 2 * pad_h
+            # for a given cell, we first print the background, then the text, then
+            # the borders
+            # need to subtract 1 because pillow uses inclusive rect coordinates
+            right = left + colwidths[r] - 1
+            bottom = top + rowheights[b] - 1
+            draw.rectangle(
+                (left, top, right, bottom),
+                cell.background,
+            )
+            # draw content adding padding
+            cell.content.draw(draw, left=left + HORIZONTAL_PADDING, top=top + VERTICAL_PADDING)
+            # draw bottom-right border
+            draw.line([
+                (left, bottom),
+                (right, bottom),
+                (right, top),
+            ], fill=(172, 176, 170))
+            if not cell.attached:
+                # overdraw previous cell's bottom border
+                draw.line([(left, top-1), (right-1, top-1)], fill=ERROR)
 
-        if r is None:  # branch cell in row
-            left = 0
-            bottom += h
-            if b:
-                draw.rectangle(
-                    (left + 1, bottom - h + 1, left + w - 1, bottom - 1),
-                    background,
-                )
-                draw.text(
-                    (left + pad_w, bottom - h + pad_h),
-                    b.name,
-                    fill=blend(TEXT, opacity, over=background),
-                )
-        elif b is None:  # repo cell in top row
-            draw.rectangle((left + 1, bottom - h + 1, left + w - 1, bottom - 1), background)
-            draw.text((left + pad_w, bottom - h + pad_h), r.name, fill=TEXT, font=bold)
-        # draw the bottom-right edges of the cell
-        draw.line([
-            (left, bottom),  # bottom-left
-            (left + w, bottom),  # bottom-right
-            (left + w, bottom - h)  # top-right
-        ], fill=(172, 176, 170))
-        if r is None or b is None:
-            continue
-
-        ps = batches[r, b]
-
-        bgcolor = BG[ps['state']]
-        if pr in ps['pr_ids']:
-            bgcolor = lighten(bgcolor, by=-0.05)
-        background = blend(bgcolor, opacity, over=background)
-        draw.rectangle((left + 1, bottom - h + 1, left + w - 1, bottom - 1), background)
-
-        top = bottom - h + pad_h
-        offset = left + pad_w
-        for p in ps['prs']:
-            label = f"#{p['number']}"
-            foreground = blend((39, 110, 114), opacity, over=background)
-            draw.text((offset, top), label, fill=foreground)
-            x, _, ww, hh = font.getbbox(label)
-            if p['closed']:
-                draw.line([
-                    (offset + x, top + hh - hh / 3),
-                    (offset + x + ww, top + hh - hh / 3),
-                ], fill=foreground)
-            offset += ww
-            if not p['attached']:
-                # overdraw top border to mark the detachment
-                draw.line([(left, bottom - h), (left + w, bottom - h)], fill=ERROR)
-            for attribute in filter(None, [
-                'error' if p['pr'].error else '',
-                '' if p['checked'] else 'missing statuses',
-                '' if p['reviewed'] else 'missing r+',
-                '' if p['attached'] else 'detached',
-                'staged' if p['pr'].staging_id else 'ready' if p['pr']._ready else ''
-            ]):
-                color = SUCCESS if attribute in ('staged', 'ready') else ERROR
-                label = f' {attribute}'
-                draw.text((offset, top), label,
-                          fill=blend(color, opacity, over=background),
-                          font=supfont)
-                offset += supfont.getbbox(label)[2]
-            offset += math.ceil(supfont.getlength(" "))
+            left += colwidths[r]
+        top += rowheights[b]
 
     return im
 

From a0d4bb0d0c8e22f8ff3cbc7619d3f6fd05bc41b0 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 26 Jul 2024 09:42:31 +0200
Subject: [PATCH 171/192] [ADD] runbot_merge: expanded view for current batch

The PR dashboard picture provides a great overview of the batch state
both horizontally and vertically *but* apparently people can't for the
life of them go check the actual dashboard when things don't line
up. So expand the "current batch" to a view more similar to
dashboard *page*, which gives details of the sub-checks being
performed and whether they are or are not fulfilled.

Fixes #908
---
 runbot_merge/controllers/dashboard.py | 107 +++++++++++++++++++++++++-
 1 file changed, 104 insertions(+), 3 deletions(-)

diff --git a/runbot_merge/controllers/dashboard.py b/runbot_merge/controllers/dashboard.py
index 7767ba97..2db8def1 100644
--- a/runbot_merge/controllers/dashboard.py
+++ b/runbot_merge/controllers/dashboard.py
@@ -238,9 +238,36 @@ class Text:
             y = top + y1 + (y2 - y1) / 3
             image.line([(left + x1, y), (left + x2, y)], self.color)
 
+@dataclass(frozen=True)
+class Checkbox:
+    checked: Optional[bool]
+    font: ImageFont.FreeTypeFont
+    color: Color
+    success: Color
+    error: Color
+
+    @cached_property
+    def width(self) -> int:
+        return ceil(max(
+            self.font.getlength(BOX_EMPTY),
+            self.font.getlength(CHECK_MARK),
+            self.font.getlength(CROSS),
+        ))
+
+    @property
+    def height(self):
+        return sum(self.font.getmetrics())
+
+    def draw(self, image: ImageDraw.ImageDraw, left: int, top: int):
+        image.text((left, top+5), BOX_EMPTY, fill=self.color, font=self.font)
+        if self.checked is True:
+            image.text((left, top+4), CHECK_MARK, fill=self.success, font=self.font)
+        elif self.checked is False:
+            image.text((left, top+4), CROSS, fill=self.error, font=self.font)
+
 @dataclass(frozen=True)
 class Line:
-    spans: List[Text]
+    spans: List[Text | Checkbox | Lines]
 
     @property
     def width(self) -> int:
@@ -294,6 +321,8 @@ def render_full_table(pr, branches, repos, batches):
         supfont = ImageFont.truetype(f, size=13, layout_engine=0)
     with file_open('web/static/fonts/google/Open_Sans/Open_Sans-Bold.ttf', 'rb') as f:
         bold = ImageFont.truetype(f, size=16, layout_engine=0)
+    with file_open('web/static/lib/fontawesome/fonts/fontawesome-webfont.ttf', 'rb') as f:
+        icons = ImageFont.truetype(f, size=16, layout_engine=0)
 
     rowheights = collections.defaultdict(int)
     colwidths = collections.defaultdict(int)
@@ -308,6 +337,80 @@ def render_full_table(pr, branches, repos, batches):
                 cell = Cell(Text("" if r is None else r.name, bold, TEXT), background)
             elif r is None: # first column
                 cell = Cell(Text(b.name, font, blend(TEXT, opacity, over=background)), background)
+            elif current_row:
+                ps = batches[r, b]
+                bgcolor = lighten(BG[ps['state']], by=-0.05) if pr in ps['pr_ids'] else BG[ps['state']]
+                background = blend(bgcolor, opacity, over=background)
+                foreground = blend((39, 110, 114), opacity, over=background)
+                success = blend(SUCCESS, opacity, over=background)
+                error = blend(ERROR, opacity, over=background)
+
+                boxes = {
+                    False: Checkbox(False, icons, foreground, success, error),
+                    True: Checkbox(True, icons, foreground, success, error),
+                    None: Checkbox(None, icons, foreground, success, error),
+                }
+                prs = []
+                attached = True
+                for p in ps['prs']:
+                    pr = p['pr']
+                    attached = attached and p['attached']
+                    sub = ": staging failed" if pr.error else ""
+                    lines = [
+                        Line([Text(
+                            f"#{p['number']}{sub}",
+                            font,
+                            foreground,
+                            decoration=Decoration.STRIKETHROUGH if p['closed'] else Decoration(0),
+                        )]),
+                    ]
+
+                    # no need for details if closed or in error
+                    if not (p['closed'] or pr.error):
+                        if pr.draft:
+                            lines.append(Line([boxes[False], Text("is in draft", font, error)]))
+                        lines.extend([
+                            Line([
+                                boxes[bool(pr.squash or pr.merge_method)],
+                                Text(
+                                    "merge method: {}".format('single' if pr.squash else (pr.merge_method or 'missing')),
+                                    font,
+                                    foreground if pr.squash or pr.merge_method else error,
+                                ),
+                            ]),
+                            Line([
+                                boxes[bool(pr.reviewed_by)],
+                                Text(
+                                    "Reviewed" if pr.reviewed_by else "Not Reviewed",
+                                    font,
+                                    foreground if pr.reviewed_by else error,
+                                )
+                            ]),
+                            Line([
+                                boxes[pr.batch_id.skipchecks or pr.status == 'success'],
+                                Text("CI", font, foreground if pr.batch_id.skipchecks or pr.status == 'success' else error),
+                            ]),
+                        ])
+                        if not pr.batch_id.skipchecks:
+                            statuses = json.loads(pr.statuses_full)
+                            for ci in pr.repository.status_ids._for_pr(pr):
+                                st = (statuses.get(ci.context.strip()) or {'state': 'pending'})['state']
+                                color = foreground
+                                if st in ('error', 'failure'):
+                                    color = error
+                                    box = boxes[False]
+                                elif st == 'success':
+                                    box = boxes[True]
+                                else:
+                                    box = boxes[None]
+
+                                lines.append(Line([
+                                    Text(" - ", font, color),
+                                    box,
+                                    Text(f"{ci.repo_id.name}: {ci.context}", font, color)
+                                ]))
+                    prs.append(Lines(lines))
+                cell = Cell(Line(prs), background, attached)
             else:
                 ps = batches[r, b]
                 bgcolor = lighten(BG[ps['state']], by=-0.05) if pr in ps['pr_ids'] else BG[ps['state']]
@@ -418,8 +521,6 @@ BG: Mapping[str | None, Color] = collections.defaultdict(lambda: (255, 255, 255)
 CHECK_MARK = "\uf00c"
 CROSS = "\uf00d"
 BOX_EMPTY = "\uf096"
-DARK_BOX = "\uf0c8"
-DARK_CHECK = "\uf14a"
 
 
 def blend_single(c: int, over: int, opacity: float) -> int:

From 6f0aea799fca0ad3f764065a376ea0f7b0bd5959 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 26 Jul 2024 12:05:51 +0200
Subject: [PATCH 172/192] [IMP] *: add test for r- on forward ports

Apparently I'd already fixed that in
286c1fdaee12bce073c28a48e4e415e843de545f but it has yet to be
deployed.

While at it, add a feedback message to clarify that, unlike `r+`, `r-`
on forward ports does *not* propagate.

Fixes #912
---
 forwardport/tests/test_simple.py     | 63 ++++++++++++++++++++++++++++
 runbot_merge/models/pull_requests.py |  4 ++
 2 files changed, 67 insertions(+)

diff --git a/forwardport/tests/test_simple.py b/forwardport/tests/test_simple.py
index b17f83d8..65bfca2d 100644
--- a/forwardport/tests/test_simple.py
+++ b/forwardport/tests/test_simple.py
@@ -553,6 +553,69 @@ def signoff(conf, message):
             return signoff
     raise AssertionError("Failed to find signoff by %s in %s" % (conf, message))
 
+def test_disapproval(env, config, make_repo, users):
+    """The author of a source PR should be able to unapprove the forward port in
+    case they approved it then noticed an issue of something.
+    """
+    # region setup
+    prod, _ = make_basic(env, config, make_repo, statuses='default')
+    env['res.partner'].create({
+        'name': users['other'],
+        'github_login': users['other'],
+        'email': 'other@example.org',
+    })
+
+    author_token = config['role_other']['token']
+    fork = prod.fork(token=author_token)
+    with prod, fork:
+        [c] = fork.make_commits('a', Commit('c_0', tree={'y': '0'}), ref='heads/accessrights')
+        pr0 = prod.make_pr(
+            target='a', title='my change',
+            head=users['other'] + ':accessrights',
+            token=author_token,
+        )
+        prod.post_status(c, 'success')
+        pr0.post_comment('hansen r+', token=config['role_reviewer']['token'])
+    env.run_crons()
+
+    with prod:
+        prod.post_status('staging.a', 'success')
+    env.run_crons()
+
+    pr0_id, pr1_id = env['runbot_merge.pull_requests'].search([], order='number')
+    assert pr1_id.source_id == pr0_id
+    pr1 = prod.get_pr(pr1_id.number)
+    assert pr0_id.state == 'merged'
+    with prod:
+        prod.post_status(pr1_id.head, 'success')
+    env.run_crons()
+    # endregion
+
+    _, _, pr2_id = env['runbot_merge.pull_requests'].search([], order='number')
+    pr2 = prod.get_pr(pr2_id.number)
+    with prod:
+        prod.post_status(pr2_id.head, 'success')
+        pr2.post_comment('hansen r+', token=config['role_other']['token'])
+    # no point creating staging for our needs, just propagate statuses
+    env.run_crons('runbot_merge.process_updated_commits')
+    assert pr1_id.state == 'ready'
+    assert pr2_id.state == 'ready'
+
+    # oh no, pr1 has an error!
+    with prod:
+        pr1.post_comment('hansen r-', token=config['role_other']['token'])
+    env.run_crons('runbot_merge.feedback_cron')
+    assert pr1_id.state == 'validated', "pr1 should not be approved anymore"
+    assert pr2_id.state == 'ready', "pr2 should not be affected"
+
+    assert pr1.comments == [
+        seen(env, pr1, users),
+        (users['user'], 'This PR targets b and is part of the forward-port chain. Further PRs will be created up to c.\n\nMore info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port\n'),
+        (users['other'], "hansen r-"),
+        (users['user'], "Note that only this forward-port has been unapproved, "
+                        "sibling forward ports may have to be unapproved "
+                        "individually."),
+    ]
 
 def test_delegate_fw(env, config, make_repo, users):
     """If a user is delegated *on a forward port* they should be able to approve
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 054b3025..c964c90d 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -831,6 +831,10 @@ For your own safety I've ignored *everything in your entire comment*.
                                 pull_request=self.number,
                                 format_args={'user': login, 'pr': self},
                             )
+                        if self.source_id:
+                            feedback("Note that only this forward-port has been"
+                                     " unapproved, sibling forward ports may "
+                                     "have to be unapproved individually.")
                         self.unstage("unreviewed (r-) by %s", login)
                     else:
                         msg = "r- makes no sense in the current PR state."

From cabab210dee5229d5ea3e2686f1ca313bd220c2b Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 26 Jul 2024 14:48:59 +0200
Subject: [PATCH 173/192] [FIX] *: don't send merge errors to logging

Merge errors are logical failures, not technical, it doesn't make
sense to log them out because there's nothing to be done technically,
a PR having consistency issues or a conflict is "normal". As such
those messages are completely useless and just take unnecessary space
in the logs, making their use more difficult.

Instead of sending them to logging, log staging attempts to the PR
itself, and only do normal logging of the operation as an indicative
item. And remove a bunch of `expect_log_errors` which don't stand
anymore.

While at it, fix a missed issue in forward porting: if the `root.head`
doesn't exist in the repo its `fetch` will immediately fail before
`cat-file` can even run, so the second one is redundant and the first
one needs to be handled properly. Do that. And leave checking
for *that* specific condition as a logged-out error as it should mean
there's something very odd with the repository (how can a pull request
have a head we can't fetch?)
---
 forwardport/models/project.py          | 21 +++++++++++----------
 forwardport/tests/test_conflicts.py    |  1 -
 forwardport/tests/test_limit.py        |  1 -
 runbot_merge/github.py                 |  2 +-
 runbot_merge/models/pull_requests.py   |  5 ++---
 runbot_merge/models/stagings_create.py |  6 ++++--
 runbot_merge/tests/test_basic.py       |  5 -----
 runbot_merge/tests/test_multirepo.py   |  1 -
 runbot_merge/tests/test_oddities.py    |  1 -
 9 files changed, 18 insertions(+), 25 deletions(-)

diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 31e0ec9c..e6599975 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -299,23 +299,24 @@ class PullRequests(models.Model):
             "Forward-porting %s (%s) to %s",
             self.display_name, root.display_name, target_branch.name
         )
-        tree = source.with_config(stdout=subprocess.PIPE, stderr=subprocess.STDOUT).fetch()
-        logger.info("Updated cache repo %s:\n%s", source._directory, tree.stdout.decode())
+        fetch = source.with_config(stdout=subprocess.PIPE, stderr=subprocess.STDOUT).fetch()
+        logger.info("Updated cache repo %s:\n%s", source._directory, fetch.stdout.decode())
 
-        tree = source.with_config(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) \
+        head_fetch = source.with_config(stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) \
             .fetch(git.source_url(self.repository), root.head)
-        logger.info(
-            "Fetched head of %s (%s):\n%s",
-            root.display_name,
-            root.head,
-            tree.stdout.decode()
-        )
-        if source.check(False).cat_file(e=root.head).returncode:
+        if head_fetch.returncode:
             raise ForwardPortError(
                 f"During forward port of {self.display_name}, unable to find "
                 f"expected head of {root.display_name} ({root.head})"
             )
 
+        logger.info(
+            "Fetched head of %s (%s):\n%s",
+            root.display_name,
+            root.head,
+            head_fetch.stdout.decode()
+        )
+
         try:
             return None, root._cherry_pick(source, target_branch.name)
         except CherrypickError as e:
diff --git a/forwardport/tests/test_conflicts.py b/forwardport/tests/test_conflicts.py
index 7011b1b3..3c0bbda6 100644
--- a/forwardport/tests/test_conflicts.py
+++ b/forwardport/tests/test_conflicts.py
@@ -383,7 +383,6 @@ def test_multiple_commits_same_authorship(env, config, make_repo):
     assert get(c.committer) == get(committer)
 
 
-@pytest.mark.expect_log_errors(reason="commits with incomplete authorship are unmergeable, which is logged as error")
 def test_multiple_commits_different_authorship(env, config, make_repo, users, rolemap):
     """ When a PR has multiple commits by different authors, the resulting
     (squashed) conflict commit should have an empty email
diff --git a/forwardport/tests/test_limit.py b/forwardport/tests/test_limit.py
index 1e8fcf7b..de563e3d 100644
--- a/forwardport/tests/test_limit.py
+++ b/forwardport/tests/test_limit.py
@@ -76,7 +76,6 @@ def test_ignore(env, config, make_repo, users):
         (users['user'], "Disabled forward-porting."),
     ]
 
-@pytest.mark.expect_log_errors(reason="one of the limit-setting does not provide a branch name")
 def test_disable(env, config, make_repo, users):
     """ Checks behaviour if the limit target is disabled:
 
diff --git a/runbot_merge/github.py b/runbot_merge/github.py
index 2abcc495..98d2e4a3 100644
--- a/runbot_merge/github.py
+++ b/runbot_merge/github.py
@@ -253,7 +253,7 @@ class GH(object):
                 raise exceptions.FastForwardError(self._repo) \
                     from Exception("timeout: never saw %s" % sha)
         except requests.HTTPError as e:
-            _logger.debug('fast_forward(%s, %s, %s) -> ERROR', self._repo, branch, sha, exc_info=True)
+            _logger.debug('fast_forward(%s, %s, %s) -> %s', self._repo, branch, sha, e)
             if e.response.status_code == 422:
                 try:
                     r = e.response.json()
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index c964c90d..a5870cdc 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -734,7 +734,6 @@ class PullRequests(models.Model):
                 e,
                 login, name,
                 utils.shorten(comment['body'] or '', 50),
-                exc_info=True
             )
             feedback(message=f"""@{login} {e.args[0]}.
 
@@ -2284,9 +2283,9 @@ class Stagings(models.Model):
                     self._safety_dance(gh, self.commits)
             except exceptions.FastForwardError as e:
                 logger.warning(
-                    "Could not fast-forward successful staging on %s:%s",
+                    "Could not fast-forward successful staging on %s:%s: %s",
                     e.args[0], self.target.name,
-                    exc_info=True
+                    e,
                 )
                 self.write({
                     'state': 'ff_failed',
diff --git a/runbot_merge/models/stagings_create.py b/runbot_merge/models/stagings_create.py
index 2005416b..55955a0a 100644
--- a/runbot_merge/models/stagings_create.py
+++ b/runbot_merge/models/stagings_create.py
@@ -259,7 +259,8 @@ def stage_batches(branch: Branch, batches: Batch, staging_state: StagingState) -
             staged |= stage_batch(env, batch, staging_state)
         except exceptions.MergeError as e:
             pr = e.args[0]
-            _logger.info("Failed to stage %s into %s", pr.display_name, branch.name, exc_info=True)
+            _logger.info("Failed to stage %s into %s", pr.display_name, branch.name)
+            pr._message_log(body=f"Failed to stage into {branch.name}: {e}")
             if not staged or isinstance(e, exceptions.Unmergeable):
                 if len(e.args) > 1 and e.args[1]:
                     reason = e.args[1]
@@ -338,7 +339,8 @@ def stage_batch(env: api.Environment, batch: Batch, staging: StagingState):
                 list(format_for_difflib((n, v) for n, v, _ in e.args[1])),
                 list(format_for_difflib((n, v) for n, _, v in e.args[1])),
             ))
-            _logger.info("data mismatch on %s:\n%s", pr.display_name, diff)
+            _logger.info("Failed to stage %s: data mismatch", pr.display_name)
+            pr._message_log(body=f"data mismatch before merge:\n{diff}")
             env.ref('runbot_merge.pr.staging.mismatch')._send(
                 repository=pr.repository,
                 pull_request=pr.number,
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index c4be7fc5..abb03ec4 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -492,7 +492,6 @@ def test_staging_concurrent(env, repo, config):
     assert pr2.staging_id
 
 
-@pytest.mark.expect_log_errors(reason="staging merge conflicts are logged")
 def test_staging_conflict_first(env, repo, users, config, page):
     """ If the first batch of a staging triggers a conflict, the PR should be
     marked as in error
@@ -524,7 +523,6 @@ def test_staging_conflict_first(env, repo, users, config, page):
     assert dangerbox[0].text.strip() == 'Unable to stage PR'
 
 
-@pytest.mark.expect_log_errors(reason="merge conflicts are logged as errors")
 def test_staging_conflict_second(env, repo, users, config):
     """ If the non-first batch of a staging triggers a conflict, the PR should
     just be skipped: it might be a conflict with an other PR which could fail
@@ -648,7 +646,6 @@ def test_staging_ci_failure_single(env, repo, users, config, page):
     assert dangerbox[0].text == 'ci/runbot'
 
 
-@pytest.mark.expect_log_errors(reason="failed fast forward of staging is a logged error")
 def test_ff_failure(env, repo, config, page):
     """ target updated while the PR is being staged => redo staging """
     with repo:
@@ -695,7 +692,6 @@ def test_ff_failure(env, repo, config, page):
         "PR should be staged to a new commit"
 
 
-@pytest.mark.expect_log_errors(reason="blocking fast-forward of staging which triggers a logged error when trying to patch the GH ref")
 def test_ff_failure_batch(env, repo, users, config):
     with repo:
         m = repo.make_commit(None, 'initial', None, tree={'m': 'm'})
@@ -3615,7 +3611,6 @@ class TestRecognizeCommands:
             prx.post_comment('%shansen r+' % indent, config['role_reviewer']['token'])
         assert pr.state == 'approved'
 
-    @pytest.mark.expect_log_errors(reason="unknown commands are logged")
     def test_unknown_commands(self, repo, env, config, users):
         with repo:
             m = repo.make_commit(None, 'initial', None, tree={'m': 'm'})
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 35a6bcbc..3ac35a8f 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -397,7 +397,6 @@ def test_sub_match(env, project, repo_a, repo_b, repo_c, config):
     assert s == list(st.ids)
 
 
-@pytest.mark.expect_log_errors(reason="merge errors are logged as errors")
 def test_merge_fail(env, project, repo_a, repo_b, users, config):
     """ In a matched-branch scenario, if merging in one of the linked repos
     fails it should revert the corresponding merges
diff --git a/runbot_merge/tests/test_oddities.py b/runbot_merge/tests/test_oddities.py
index 1454165b..6bee3696 100644
--- a/runbot_merge/tests/test_oddities.py
+++ b/runbot_merge/tests/test_oddities.py
@@ -173,7 +173,6 @@ def test_merge_empty_commits(env, repo, users, config):
     assert commits[2]['commit']['message'] == 'initial'
 
 
-@pytest.mark.expect_log_errors(reason="emptied commits are merge errors and logged")
 def test_merge_emptying_commits(env, repo, users, config):
     """The mergebot should *not* allow merging non-empty commits which become
     empty as part of the staging (rebasing)

From 48d22b920ad2bdab18db1d6a388c8d7de43e1c88 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 30 Jul 2024 11:04:45 +0200
Subject: [PATCH 174/192] [FIX] mergebot coverage

- add context, in case that allows tracking overlap between tests
  eventually
- use `-m` to run odoo, requires backporting `odoo/__main__.py`
  but I'm not quite sure how I was running it previously, possibly
  via odoo-bin? It certainly doesn't seem to work out with a global
  `odoo` helper
---
 conftest.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/conftest.py b/conftest.py
index 66afa127..610a75d3 100644
--- a/conftest.py
+++ b/conftest.py
@@ -451,7 +451,13 @@ def server(request, db, port, module, addons_path, tmpdir):
 
     cov = []
     if request.config.getoption('--coverage'):
-        cov = ['coverage', 'run', '-p', '--source=odoo.addons.runbot_merge,odoo.addons.forwardport', '--branch']
+        cov = [
+            'coverage', 'run',
+            '-p', '--branch',
+            '--source=odoo.addons.runbot_merge,odoo.addons.forwardport',
+            '--context', request.node.nodeid,
+            '-m',
+        ]
 
     r, w = os.pipe2(os.O_NONBLOCK)
     buf = bytearray()

From 32871c389690923aeb8a06703977af0fdec9c7fe Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 30 Jul 2024 13:47:49 +0200
Subject: [PATCH 175/192] [FIX] *: stray prints

Found a bunch of old leftover `print` calls which should not be in the
repo.
---
 conftest.py                     | 1 -
 forwardport/models/project.py   | 2 --
 forwardport/tests/test_weird.py | 5 -----
 3 files changed, 8 deletions(-)

diff --git a/conftest.py b/conftest.py
index 66afa127..7fb85dde 100644
--- a/conftest.py
+++ b/conftest.py
@@ -602,7 +602,6 @@ def _rate_limited(req):
         if not q.ok and q.headers.get('X-RateLimit-Remaining') == '0':
             reset = int(q.headers['X-RateLimit-Reset'])
             delay = max(0, round(reset - time.time() + 1.0))
-            print("Hit rate limit, sleeping for", delay, "seconds")
             time.sleep(delay)
             continue
         break
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index e6599975..30c2df5d 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -501,8 +501,6 @@ stderr:
             pr_ids = {p.id for p in prs}
             for pr in prs:
                 pr_ids.discard(pr.parent_id.id)
-            print(source, prs, [p.parent_id for p in prs],
-                  '\n\t->', pr_ids, flush=True)
             for pr in (p for p in prs if p.id in pr_ids):
                 self.env.ref('runbot_merge.forwardport.reminder')._send(
                     repository=pr.repository,
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index e2c573b6..47cf970a 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -1197,11 +1197,6 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
     pr_c_id.write({'parent_id': False, 'detach_reason': 'because'})
     env.run_crons('forwardport.reminder', 'runbot_merge.feedback_cron', context={'forwardport_updated_before': FAKE_PREV_WEEK})
 
-    for p in env['runbot_merge.pull_requests'].search_read(
-        [], ['id', 'parent_id', 'source_id', 'display_name']
-    ):
-        print(p, flush=True)
-
     assert pr_b.comments[2:] == [
         (users['user'], "@%s @%s child PR %s was modified / updated and has become a normal PR. This PR (and any of its parents) will need to be merged independently as approvals won't cross." % (
             users['user'],

From dd17730f4cd3c24ea60bc27282a5badec9738cfc Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 30 Jul 2024 09:21:05 +0200
Subject: [PATCH 176/192] [IMP] *: crons tests running for better triggered
 compatibility

Mergebot / forwardport crons need to run in a specific ordering in
order to flow into one another correctly. The default ordering being
unspecified, it was not possible to use the normal cron
runner (instead of the external driver running crons in sequence one
at a time). This can be fixed by setting *sequences* on crons, as the
cron runner (`_process_jobs`) will use that order to acquire and run
crons.

Also override `_process_jobs` however: the built-in cron runner
fetches a static list of ready crons, then runs that.

This is fine for normal situation where the cron runner runs in a loop
anyway but it's any issue for the tests, as we expect that cron A can
trigger cron B, and we want cron B to run *right now* even if it
hadn't been triggered before cron A ran.

We can replace `_process_job` with a cut down version which does
that (cut down because we don't need most of the error handling /
resilience, there's no concurrent workers, there's no module being
installed, versions must match, ...). This allows e.g. the cron
propagating commit statuses to trigger the staging cron, and both will
run within the same `run_crons` session.

Something I didn't touch is that `_process_jobs` internally creates
completely new environments so there is no way to pass context into
the cron jobs anymore (whereas it works for `method_direct_trigger`),
this means the context values have to be shunted elsewhere for that
purpose which is gross. But even though I'm replacing `_process_jobs`,
this seems a bit too much of a change in cron execution semantics. So
left it out.

While at it tho, silence the spammy `py.warnings` stuff I can't do
much about.
---
 conftest.py                                   | 64 +++++++++++++++++--
 forwardport/data/crons.xml                    |  2 +
 forwardport/models/forwardport.py             |  3 +-
 forwardport/models/project.py                 |  3 +-
 runbot_merge/data/merge_cron.xml              |  7 ++
 .../models/crons/cleanup_scratch_branches.py  |  2 +-
 runbot_merge/tests/test_disabled_branch.py    | 22 ++++---
 runbot_merge/tests/test_project_toggles.py    | 16 +++--
 8 files changed, 96 insertions(+), 23 deletions(-)

diff --git a/conftest.py b/conftest.py
index 7fb85dde..59f0753e 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import datetime
 import select
 import shutil
 import threading
@@ -316,7 +317,8 @@ class DbDict(dict):
                 '-d', db, '-i', module + ',saas_worker,auth_oauth',
                 '--max-cron-threads', '0',
                 '--stop-after-init',
-                '--log-level', 'warn'
+                '--log-level', 'warn',
+                '--log-handler', 'py.warnings:ERROR',
             ],
                 check=True,
                 env={**os.environ, 'XDG_DATA_HOME': str(d)}
@@ -413,15 +415,58 @@ def dummy_addons_path():
         mod = pathlib.Path(dummy_addons_path, 'saas_worker')
         mod.mkdir(0o700)
         (mod / '__init__.py').write_text('''\
+import builtins
+import logging
+import threading
+
+import psycopg2
+
+import odoo
 from odoo import api, fields, models
 
+_logger = logging.getLogger(__name__)
+
 
 class Base(models.AbstractModel):
     _inherit = 'base'
 
     def run_crons(self):
+        builtins.forwardport_merged_before = self.env.context.get('forwardport_merged_before')
+        builtins.forwardport_updated_before = self.env.context.get('forwardport_updated_before')
         self.env['ir.cron']._process_jobs(self.env.cr.dbname)
+        del builtins.forwardport_updated_before
+        del builtins.forwardport_merged_before
         return True
+
+
+class IrCron(models.Model):
+    _inherit = 'ir.cron'
+
+    @classmethod
+    def _process_jobs(cls, db_name):
+        t = threading.current_thread()
+        try:
+            db = odoo.sql_db.db_connect(db_name)
+            t.dbname = db_name
+            with db.cursor() as cron_cr:
+                # FIXME: override `_get_all_ready_jobs` to directly lock the cron?
+                while jobs := next((
+                    job
+                    for j in cls._get_all_ready_jobs(cron_cr)
+                    if (job := cls._acquire_one_job(cron_cr, (j['id'],)))
+                ), None):
+                    # take into account overridings of _process_job() on that database
+                    registry = odoo.registry(db_name)
+                    registry[cls._name]._process_job(db, cron_cr, job)
+                    cron_cr.commit()
+
+        except psycopg2.ProgrammingError as e:
+            raise
+        except Exception:
+            _logger.warning('Exception in cron:', exc_info=True)
+        finally:
+            if hasattr(t, 'dbname'):
+                del t.dbname
 ''', encoding='utf-8')
         (mod / '__manifest__.py').write_text(pprint.pformat({
             'name': 'dummy saas_worker',
@@ -445,6 +490,7 @@ def addons_path(request, dummy_addons_path):
 def server(request, db, port, module, addons_path, tmpdir):
     log_handlers = [
         'odoo.modules.loading:WARNING',
+        'py.warnings:ERROR',
     ]
     if not request.config.getoption('--log-github'):
         log_handlers.append('github_requests:WARNING')
@@ -1210,15 +1256,19 @@ class Environment:
 
     def run_crons(self, *xids, **kw):
         crons = xids or self._default_crons
-        print('running crons', crons, file=sys.stderr)
+        cron_ids = []
         for xid in crons:
-            t0 = time.time()
-            print('\trunning cron', xid, '...', file=sys.stderr)
+            if xid is None:
+                continue
+
             model, cron_id = self('ir.model.data', 'check_object_reference', *xid.split('.', 1))
             assert model == 'ir.cron', "Expected {} to be a cron, got {}".format(xid, model)
-            self('ir.cron', 'method_direct_trigger', [cron_id], **kw)
-            print('\tdone %.3fs' % (time.time() - t0), file=sys.stderr)
-        print('done', file=sys.stderr)
+            cron_ids.append(cron_id)
+        if cron_ids:
+            self('ir.cron', 'write', cron_ids, {
+                'nextcall': (datetime.datetime.utcnow() - datetime.timedelta(seconds=30)).isoformat(" ", "seconds")
+            }, **kw)
+        self('base', 'run_crons', [], **kw)
         # sleep for some time as a lot of crap may have happened (?)
         wait_for_hook()
 
diff --git a/forwardport/data/crons.xml b/forwardport/data/crons.xml
index 02d2be1e..7d0e1bf7 100644
--- a/forwardport/data/crons.xml
+++ b/forwardport/data/crons.xml
@@ -8,6 +8,7 @@
         <field name="interval_type">minutes</field>
         <field name="numbercall">-1</field>
         <field name="doall" eval="False"/>
+        <field name="priority">43</field>
     </record>
 
     <record model="ir.cron" id="updates">
@@ -19,6 +20,7 @@
         <field name="interval_type">minutes</field>
         <field name="numbercall">-1</field>
         <field name="doall" eval="False"/>
+        <field name="priority">46</field>
     </record>
 
     <record model="ir.cron" id="reminder">
diff --git a/forwardport/models/forwardport.py b/forwardport/models/forwardport.py
index a0604da7..b6ac07d8 100644
--- a/forwardport/models/forwardport.py
+++ b/forwardport/models/forwardport.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import builtins
 import contextlib
 import logging
 import re
@@ -345,7 +346,7 @@ class DeleteBranches(models.Model, Queue):
     pr_id = fields.Many2one('runbot_merge.pull_requests')
 
     def _search_domain(self):
-        cutoff = self.env.context.get('forwardport_merged_before') \
+        cutoff = getattr(builtins, 'forwardport_merged_before', None) \
              or fields.Datetime.to_string(datetime.now() - MERGE_AGE)
         return [('pr_id.merge_date', '<', cutoff)]
 
diff --git a/forwardport/models/project.py b/forwardport/models/project.py
index 30c2df5d..72630453 100644
--- a/forwardport/models/project.py
+++ b/forwardport/models/project.py
@@ -13,6 +13,7 @@ it up), ...
 """
 from __future__ import annotations
 
+import builtins
 import datetime
 import itertools
 import json
@@ -487,7 +488,7 @@ stderr:
         ], order='source_id, id'), lambda p: p.source_id)
 
     def _reminder(self):
-        cutoff = self.env.context.get('forwardport_updated_before') \
+        cutoff = getattr(builtins, 'forwardport_updated_before', None) \
               or fields.Datetime.to_string(datetime.datetime.now() - DEFAULT_DELTA)
         cutoff_dt = fields.Datetime.from_string(cutoff)
 
diff --git a/runbot_merge/data/merge_cron.xml b/runbot_merge/data/merge_cron.xml
index b0ff6e1d..ffd3147f 100644
--- a/runbot_merge/data/merge_cron.xml
+++ b/runbot_merge/data/merge_cron.xml
@@ -8,6 +8,7 @@
     <field name="interval_type">minutes</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
+    <field name="priority">30</field>
   </record>
   <record model="ir.cron" id="staging_cron">
     <field name="name">Check for progress of PRs and create Stagings</field>
@@ -18,6 +19,7 @@
     <field name="interval_type">minutes</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
+    <field name="priority">40</field>
   </record>
   <record model="ir.cron" id="feedback_cron">
     <field name="name">Send feedback to PR</field>
@@ -28,6 +30,7 @@
     <field name="interval_type">minutes</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
+    <field name="priority">60</field>
   </record>
   <record model="ir.cron" id="labels_cron">
     <field name="name">Update labels on PR</field>
@@ -38,6 +41,7 @@
     <field name="interval_type">minutes</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
+    <field name="priority">70</field>
   </record>
   <record model="ir.cron" id="fetch_prs_cron">
     <field name="name">Check for PRs to fetch</field>
@@ -48,6 +52,7 @@
     <field name="interval_type">minutes</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
+    <field name="priority">10</field>
   </record>
   <record model="ir.cron" id="check_linked_prs_status">
     <field name="name">Warn on linked PRs where only one is ready</field>
@@ -58,6 +63,7 @@
     <field name="interval_type">hours</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
+    <field name="priority">50</field>
   </record>
   <record model="ir.cron" id="process_updated_commits">
     <field name="name">Impact commit statuses on PRs and stagings</field>
@@ -68,5 +74,6 @@
     <field name="interval_type">minutes</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
+    <field name="priority">20</field>
   </record>
 </odoo>
diff --git a/runbot_merge/models/crons/cleanup_scratch_branches.py b/runbot_merge/models/crons/cleanup_scratch_branches.py
index 1e7b6274..a5e041ce 100644
--- a/runbot_merge/models/crons/cleanup_scratch_branches.py
+++ b/runbot_merge/models/crons/cleanup_scratch_branches.py
@@ -29,4 +29,4 @@ class BranchCleanup(models.TransientModel):
                     _logger.info("no tmp branch found for %s:%s", r.name, b.name)
                 res = gh('delete', f'git/refs/heads/staging.{b.name}', check=False)
                 if res.status_code != 204:
-                    _logger.info("no staging branch found for %s:%s", res.name, b.name)
+                    _logger.info("no staging branch found for %s:%s", r.name, b.name)
diff --git a/runbot_merge/tests/test_disabled_branch.py b/runbot_merge/tests/test_disabled_branch.py
index cfd16bae..88e5e490 100644
--- a/runbot_merge/tests/test_disabled_branch.py
+++ b/runbot_merge/tests/test_disabled_branch.py
@@ -45,10 +45,21 @@ def test_existing_pr_disabled_branch(env, project, make_repo, setreviewers, conf
     staging_id = branch_id.active_staging_id
     assert staging_id == pr_id.staging_id
 
+    # staging of `pr` should have generated a staging branch
+    _ = repo.get_ref('heads/staging.other')
+    # stagings should not need a tmp branch anymore, so this should not exist
+    with pytest.raises(AssertionError, match=r'Not Found'):
+        repo.get_ref('heads/tmp.other')
+
     # disable branch "other"
     branch_id.active = False
     env.run_crons()
 
+    # triggered cleanup should have deleted the staging for the disabled `other`
+    # target branch
+    with pytest.raises(AssertionError, match=r'Not Found'):
+        repo.get_ref('heads/staging.other')
+
     # the PR should not have been closed implicitly
     assert pr_id.state == 'ready'
     # but it should be unstaged
@@ -86,17 +97,10 @@ def test_existing_pr_disabled_branch(env, project, make_repo, setreviewers, conf
     assert pr_id.staging_id
 
     # staging of `pr` should have generated a staging branch
-    _ = repo.get_ref('heads/staging.other')
+    _ = repo.get_ref('heads/staging.other2')
     # stagings should not need a tmp branch anymore, so this should not exist
     with pytest.raises(AssertionError, match=r'Not Found'):
-        repo.get_ref('heads/tmp.other')
-
-    assert env['base'].run_crons()
-
-    # triggered cleanup should have deleted the staging for the disabled `other`
-    # target branch
-    with pytest.raises(AssertionError, match=r'Not Found'):
-        repo.get_ref('heads/staging.other')
+        repo.get_ref('heads/tmp.other2')
 
 def test_new_pr_no_branch(env, project, make_repo, setreviewers, users):
     """ A new PR to an *unknown* branch should be ignored and warn
diff --git a/runbot_merge/tests/test_project_toggles.py b/runbot_merge/tests/test_project_toggles.py
index 615e6e4b..30fe4eff 100644
--- a/runbot_merge/tests/test_project_toggles.py
+++ b/runbot_merge/tests/test_project_toggles.py
@@ -1,3 +1,4 @@
+import datetime
 import functools
 from itertools import repeat
 
@@ -66,16 +67,16 @@ def test_staging_priority(env, project, repo, config, mode, cutoff, second):
     with repo:
         [m] = repo.make_commits(None, Commit("m", tree={"ble": "1"}), ref="heads/master")
 
-        [c] = repo.make_commits(m, Commit("c", tree={"1": "1"}), ref="heads/pr1")
+        repo.make_commits(m, Commit("c", tree={"1": "1"}), ref="heads/pr1")
         pr1 = repo.make_pr(title="whatever", target="master", head="pr1")
 
-        [c] = repo.make_commits(m, Commit("c", tree={"2": "2"}), ref="heads/pr2")
+        repo.make_commits(m, Commit("c", tree={"2": "2"}), ref="heads/pr2")
         pr2 = repo.make_pr(title="whatever", target="master", head="pr2")
 
-        [c] = repo.make_commits(m, Commit("c", tree={"3": "3"}), ref="heads/pr3")
+        repo.make_commits(m, Commit("c", tree={"3": "3"}), ref="heads/pr3")
         pr3 = repo.make_pr(title="whatever", target="master", head="pr3")
 
-        [c] = repo.make_commits(m, Commit("c", tree={"4": "4"}), ref="heads/pr4")
+        repo.make_commits(m, Commit("c", tree={"4": "4"}), ref="heads/pr4")
         pr4 = repo.make_pr(title="whatever", target="master", head="pr4")
 
     prs = [pr1, pr2, pr3, pr4]
@@ -106,6 +107,13 @@ def test_staging_priority(env, project, repo, config, mode, cutoff, second):
     # trigger a split
     with repo:
         repo.post_status('staging.master', 'failure')
+
+    # specifically delay creation of new staging to observe the failed
+    # staging's state and the splits
+    model, cron_id = env['ir.model.data'].check_object_reference('runbot_merge', 'staging_cron')
+    env[model].browse([cron_id]).write({
+        'nextcall': (datetime.datetime.utcnow() + datetime.timedelta(minutes=10)).isoformat(" ", "seconds")
+    })
     env.run_crons('runbot_merge.process_updated_commits', 'runbot_merge.merge_cron')
     assert not staging.active
     assert not env['runbot_merge.stagings'].search([]).active

From 57a82235d9694bf6494b3b4090abfdee285307bc Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 30 Jul 2024 13:42:00 +0200
Subject: [PATCH 177/192] [IMP] *: rely only on triggers to run statuses
 propagation

The cron had been converted to using mostly triggers in
7cd9afe7f2ff2cbaaabe0ecf6eb364b55aaf8d60 but I forgot to update
the *tests* to avoid explicitly triggering it.
---
 forwardport/tests/conftest.py              |  1 -
 forwardport/tests/test_simple.py           |  2 +-
 runbot_merge/data/merge_cron.xml           |  4 ++--
 runbot_merge/tests/conftest.py             |  2 --
 runbot_merge/tests/test_basic.py           |  8 ++++----
 runbot_merge/tests/test_by_branch.py       | 20 ++++++++++----------
 runbot_merge/tests/test_project_toggles.py |  4 ++--
 7 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/forwardport/tests/conftest.py b/forwardport/tests/conftest.py
index 408345af..0a656c52 100644
--- a/forwardport/tests/conftest.py
+++ b/forwardport/tests/conftest.py
@@ -7,7 +7,6 @@ import requests
 @pytest.fixture
 def default_crons():
     return [
-        'runbot_merge.process_updated_commits',
         'runbot_merge.merge_cron',
         'runbot_merge.staging_cron',
         'forwardport.port_forward',
diff --git a/forwardport/tests/test_simple.py b/forwardport/tests/test_simple.py
index 65bfca2d..2e5a5af8 100644
--- a/forwardport/tests/test_simple.py
+++ b/forwardport/tests/test_simple.py
@@ -597,7 +597,7 @@ def test_disapproval(env, config, make_repo, users):
         prod.post_status(pr2_id.head, 'success')
         pr2.post_comment('hansen r+', token=config['role_other']['token'])
     # no point creating staging for our needs, just propagate statuses
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert pr1_id.state == 'ready'
     assert pr2_id.state == 'ready'
 
diff --git a/runbot_merge/data/merge_cron.xml b/runbot_merge/data/merge_cron.xml
index ffd3147f..3240e3ae 100644
--- a/runbot_merge/data/merge_cron.xml
+++ b/runbot_merge/data/merge_cron.xml
@@ -70,8 +70,8 @@
     <field name="model_id" ref="model_runbot_merge_commit"/>
     <field name="state">code</field>
     <field name="code">model._notify()</field>
-    <field name="interval_number">30</field>
-    <field name="interval_type">minutes</field>
+    <field name="interval_number">6</field>
+    <field name="interval_type">hours</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
     <field name="priority">20</field>
diff --git a/runbot_merge/tests/conftest.py b/runbot_merge/tests/conftest.py
index b663a688..5f84298c 100644
--- a/runbot_merge/tests/conftest.py
+++ b/runbot_merge/tests/conftest.py
@@ -9,8 +9,6 @@ def default_crons():
     return [
         # env['runbot_merge.project']._check_fetch()
         'runbot_merge.fetch_prs_cron',
-        # env['runbot_merge.commit']._notify()
-        'runbot_merge.process_updated_commits',
         # env['runbot_merge.project']._check_stagings()
         'runbot_merge.merge_cron',
         # env['runbot_merge.project']._create_stagings()
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index abb03ec4..e738787d 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -607,7 +607,7 @@ def test_timeout_bump_on_pending(env, repo, config):
     st.timeout_limit = old_timeout
     with repo:
         repo.post_status('staging.master', 'pending', 'ci/runbot')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert st.timeout_limit > old_timeout
 
 def test_staging_ci_failure_single(env, repo, users, config, page):
@@ -2463,7 +2463,7 @@ class TestPRUpdate(object):
 
         env.run_crons()
         pr_id = to_pr(env, pr)
-        env.run_crons('runbot_merge.process_updated_commits')
+        env.run_crons(None)
         assert pr_id.message == 'title\n\nbody'
         assert pr_id.state == 'ready'
         old_reviewer = pr_id.reviewed_by
@@ -3008,7 +3008,7 @@ class TestBatching(object):
                 reviewer=None,
             )
             pr02.post_comment('hansen alone r+', config['role_reviewer']['token'])
-        env.run_crons('runbot_merge.process_updated_commits')
+        env.run_crons(None)
         pr01_id = to_pr(env, pr01)
         assert pr01_id.blocked is False
         pr02_id = to_pr(env, pr02)
@@ -3077,7 +3077,7 @@ class TestBatching(object):
         with repo:
             repo.post_status('staging.master', 'success', 'ci/runbot')
             repo.post_status('staging.master', 'success', 'legal/cla')
-        env.run_crons('runbot_merge.process_updated_commits', 'runbot_merge.merge_cron', 'runbot_merge.staging_cron')
+        env.run_crons('runbot_merge.merge_cron', 'runbot_merge.staging_cron')
         assert pr2.state == 'merged'
 
 class TestReviewing:
diff --git a/runbot_merge/tests/test_by_branch.py b/runbot_merge/tests/test_by_branch.py
index efaf0394..9134a4e2 100644
--- a/runbot_merge/tests/test_by_branch.py
+++ b/runbot_merge/tests/test_by_branch.py
@@ -34,15 +34,15 @@ def test_status_applies(env, repo, config):
 
     with repo:
         repo.post_status(c, 'success', 'ci')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert pr_id.state == 'opened'
     with repo:
         repo.post_status(c, 'success', 'pr')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert pr_id.state == 'opened'
     with repo:
         repo.post_status(c, 'success', 'lint')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert pr_id.state == 'validated'
 
     with repo:
@@ -53,15 +53,15 @@ def test_status_applies(env, repo, config):
     assert st.state == 'pending'
     with repo:
         repo.post_status('staging.master', 'success', 'ci')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert st.state == 'pending'
     with repo:
         repo.post_status('staging.master', 'success', 'lint')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert st.state == 'pending'
     with repo:
         repo.post_status('staging.master', 'success', 'staging')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert st.state == 'success'
 
 @pytest.mark.usefixtures('_setup_statuses')
@@ -84,11 +84,11 @@ def test_status_skipped(env, project, repo, config):
 
     with repo:
         repo.post_status(c, 'success', 'ci')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert pr_id.state == 'opened'
     with repo:
         repo.post_status(c, 'success', 'pr')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert pr_id.state == 'validated'
 
     with repo:
@@ -99,11 +99,11 @@ def test_status_skipped(env, project, repo, config):
     assert st.state == 'pending'
     with repo:
         repo.post_status('staging.maintenance', 'success', 'staging')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert st.state == 'pending'
     with repo:
         repo.post_status('staging.maintenance', 'success', 'ci')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert st.state == 'success'
 
 def test_pseudo_version_tag(env, project, make_repo, setreviewers, config):
diff --git a/runbot_merge/tests/test_project_toggles.py b/runbot_merge/tests/test_project_toggles.py
index 30fe4eff..30ce5f78 100644
--- a/runbot_merge/tests/test_project_toggles.py
+++ b/runbot_merge/tests/test_project_toggles.py
@@ -101,7 +101,7 @@ def test_staging_priority(env, project, repo, config, mode, cutoff, second):
         for pr, pr_id in zip(prs[cutoff:], pr_ids[cutoff:]):
             pr.post_comment('hansen r+', config['role_reviewer']['token'])
             repo.post_status(pr_id.head, 'success')
-    env.run_crons('runbot_merge.process_updated_commits')
+    env.run_crons(None)
     assert not pr_ids.filtered(lambda p: p.blocked)
 
     # trigger a split
@@ -114,7 +114,7 @@ def test_staging_priority(env, project, repo, config, mode, cutoff, second):
     env[model].browse([cron_id]).write({
         'nextcall': (datetime.datetime.utcnow() + datetime.timedelta(minutes=10)).isoformat(" ", "seconds")
     })
-    env.run_crons('runbot_merge.process_updated_commits', 'runbot_merge.merge_cron')
+    env.run_crons('runbot_merge.merge_cron')
     assert not staging.active
     assert not env['runbot_merge.stagings'].search([]).active
     assert env['runbot_merge.split'].search_count([]) == 2

From 029957dbebb2039d40f55c83835e6576c463e18a Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Tue, 30 Jul 2024 13:45:51 +0200
Subject: [PATCH 178/192] [IMP] *: trigger-ify task queue type crons

These are pretty simple to convert as they are straightforward: an
item is added  to a work queue (table), then a cron regularly scans
through the table executing the items and deleting them.

That means the cron trigger can just be added on `create` and things
should work out fine.

There's just two wrinkles in the port_forward cron:

- It can be requeued in the future, so needs a conditional trigger-ing
  in `write`.
- It is disabled during freeze (maybe something to change), as a
  result triggers don't enqueue at all, so we need to immediately
  trigger after freeze to force the cron re-enabling it.
---
 forwardport/data/crons.xml           | 10 +++++-----
 forwardport/models/forwardport.py    | 18 ++++++++++++++++++
 forwardport/models/project_freeze.py |  4 +++-
 forwardport/tests/conftest.py        |  3 ---
 forwardport/tests/test_simple.py     | 12 ++++++------
 forwardport/tests/test_weird.py      | 16 ++++++----------
 runbot_merge/data/merge_cron.xml     |  8 ++++----
 runbot_merge/models/pull_requests.py | 11 +++++++++++
 runbot_merge/tests/conftest.py       |  4 ----
 runbot_merge/tests/test_basic.py     |  2 --
 10 files changed, 53 insertions(+), 35 deletions(-)

diff --git a/forwardport/data/crons.xml b/forwardport/data/crons.xml
index 7d0e1bf7..b3ce4591 100644
--- a/forwardport/data/crons.xml
+++ b/forwardport/data/crons.xml
@@ -4,8 +4,8 @@
         <field name="model_id" ref="model_forwardport_batches"/>
         <field name="state">code</field>
         <field name="code">model._process()</field>
-        <field name="interval_number">1</field>
-        <field name="interval_type">minutes</field>
+        <field name="interval_number">6</field>
+        <field name="interval_type">hours</field>
         <field name="numbercall">-1</field>
         <field name="doall" eval="False"/>
         <field name="priority">43</field>
@@ -16,8 +16,8 @@
         <field name="model_id" ref="model_forwardport_updates"/>
         <field name="state">code</field>
         <field name="code">model._process()</field>
-        <field name="interval_number">1</field>
-        <field name="interval_type">minutes</field>
+        <field name="interval_number">6</field>
+        <field name="interval_type">hours</field>
         <field name="numbercall">-1</field>
         <field name="doall" eval="False"/>
         <field name="priority">46</field>
@@ -39,7 +39,7 @@
         <field name="model_id" ref="model_forwardport_branch_remover"/>
         <field name="state">code</field>
         <field name="code">model._process()</field>
-        <field name="interval_number">1</field>
+        <field name="interval_number">6</field>
         <field name="interval_type">hours</field>
         <field name="numbercall">-1</field>
         <field name="doall" eval="False"/>
diff --git a/forwardport/models/forwardport.py b/forwardport/models/forwardport.py
index b6ac07d8..e805e59a 100644
--- a/forwardport/models/forwardport.py
+++ b/forwardport/models/forwardport.py
@@ -77,6 +77,16 @@ class ForwardPortTasks(models.Model, Queue):
     retry_after = fields.Datetime(required=True, default='1900-01-01 01:01:01')
     pr_id = fields.Many2one('runbot_merge.pull_requests')
 
+    def create(self, vals_list):
+        self.env.ref('forwardport.port_forward')._trigger()
+        return super().create(vals_list)
+
+    def write(self, vals):
+        if retry := vals.get('retry_after'):
+            self.env.ref('forwardport.port_forward')\
+                ._trigger(fields.Datetime.to_datetime(retry))
+        return super().write(vals)
+
     def _search_domain(self):
         return super()._search_domain() + [
             ('retry_after', '<=', fields.Datetime.to_string(fields.Datetime.now())),
@@ -260,6 +270,10 @@ class UpdateQueue(models.Model, Queue):
     original_root = fields.Many2one('runbot_merge.pull_requests')
     new_root = fields.Many2one('runbot_merge.pull_requests')
 
+    def create(self, vals_list):
+        self.env.ref('forwardport.updates')._trigger()
+        return super().create(vals_list)
+
     def _process_item(self):
         previous = self.new_root
         sentry_sdk.set_tag("update-root", self.new_root.display_name)
@@ -345,6 +359,10 @@ class DeleteBranches(models.Model, Queue):
 
     pr_id = fields.Many2one('runbot_merge.pull_requests')
 
+    def create(self, vals_list):
+        self.env.ref('forwardport.remover')._trigger(datetime.now() - MERGE_AGE)
+        return super().create(vals_list)
+
     def _search_domain(self):
         cutoff = getattr(builtins, 'forwardport_merged_before', None) \
              or fields.Datetime.to_string(datetime.now() - MERGE_AGE)
diff --git a/forwardport/models/project_freeze.py b/forwardport/models/project_freeze.py
index 635912c0..0796fbaf 100644
--- a/forwardport/models/project_freeze.py
+++ b/forwardport/models/project_freeze.py
@@ -22,5 +22,7 @@ class FreezeWizard(models.Model):
     def unlink(self):
         r = super().unlink()
         if not (self.env.context.get('forwardport_keep_disabled') or self.search_count([])):
-            self.env.ref('forwardport.port_forward').active = True
+            cron = self.env.ref('forwardport.port_forward')
+            cron.active = True
+            cron._trigger()  # process forward ports enqueued during the freeze period
         return r
diff --git a/forwardport/tests/conftest.py b/forwardport/tests/conftest.py
index 0a656c52..5243d96e 100644
--- a/forwardport/tests/conftest.py
+++ b/forwardport/tests/conftest.py
@@ -9,10 +9,7 @@ def default_crons():
     return [
         'runbot_merge.merge_cron',
         'runbot_merge.staging_cron',
-        'forwardport.port_forward',
-        'forwardport.updates',
         'runbot_merge.check_linked_prs_status',
-        'runbot_merge.feedback_cron',
     ]
 
 # public_repo — necessary to leave comments
diff --git a/forwardport/tests/test_simple.py b/forwardport/tests/test_simple.py
index 2e5a5af8..634a25c0 100644
--- a/forwardport/tests/test_simple.py
+++ b/forwardport/tests/test_simple.py
@@ -123,7 +123,7 @@ def test_straightforward_flow(env, config, make_repo, users):
         prod.post_status(pr1.head, 'success', 'legal/cla')
 
     env.run_crons()
-    env.run_crons('forwardport.reminder', 'runbot_merge.feedback_cron', context={'forwardport_updated_before': FAKE_PREV_WEEK})
+    env.run_crons('forwardport.reminder', context={'forwardport_updated_before': FAKE_PREV_WEEK})
 
     pr0_, pr1_, pr2 = env['runbot_merge.pull_requests'].search([], order='number')
 
@@ -329,8 +329,8 @@ def test_empty(env, config, make_repo, users):
     assert project.fp_github_name == users['other']
 
     # check reminder
-    env.run_crons('forwardport.reminder', 'runbot_merge.feedback_cron', context={'forwardport_updated_before': FAKE_PREV_WEEK})
-    env.run_crons('forwardport.reminder', 'runbot_merge.feedback_cron', context={'forwardport_updated_before': FAKE_PREV_WEEK})
+    env.run_crons('forwardport.reminder', context={'forwardport_updated_before': FAKE_PREV_WEEK})
+    env.run_crons('forwardport.reminder', context={'forwardport_updated_before': FAKE_PREV_WEEK})
 
     awaiting = (
         users['other'],
@@ -374,7 +374,7 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
     # check that this stops if we close the PR
     with prod:
         fail_pr.close()
-    env.run_crons('forwardport.reminder', 'runbot_merge.feedback_cron', context={'forwardport_updated_before': FAKE_PREV_WEEK})
+    env.run_crons('forwardport.reminder', context={'forwardport_updated_before': FAKE_PREV_WEEK})
     assert pr1.comments == [
         (users['reviewer'], 'hansen r+'),
         seen(env, pr1, users),
@@ -604,7 +604,7 @@ def test_disapproval(env, config, make_repo, users):
     # oh no, pr1 has an error!
     with prod:
         pr1.post_comment('hansen r-', token=config['role_other']['token'])
-    env.run_crons('runbot_merge.feedback_cron')
+    env.run_crons(None)
     assert pr1_id.state == 'validated', "pr1 should not be approved anymore"
     assert pr2_id.state == 'ready', "pr2 should not be affected"
 
@@ -899,7 +899,7 @@ class TestClosing:
             prod.post_status(pr1_id.head, 'success', 'legal/cla')
             prod.post_status(pr1_id.head, 'success', 'ci/runbot')
         env.run_crons()
-        env.run_crons('forwardport.reminder', 'runbot_merge.feedback_cron')
+        env.run_crons('forwardport.reminder')
 
         assert env['runbot_merge.pull_requests'].search([], order='number') == pr0_id | pr1_id,\
             "closing the PR should suppress the FP sequence"
diff --git a/forwardport/tests/test_weird.py b/forwardport/tests/test_weird.py
index 47cf970a..19feda24 100644
--- a/forwardport/tests/test_weird.py
+++ b/forwardport/tests/test_weird.py
@@ -695,7 +695,7 @@ def test_retarget_after_freeze(env, config, make_repo, users):
         port_pr.base = 'bprime'
     assert port_id.target == new_branch
 
-    env.run_crons('forwardport.port_forward')
+    env.run_crons(None)
     assert not job.exists(), "job should have succeeded and apoptosed"
 
     # since the PR was "already forward-ported" to the new branch it should not
@@ -816,11 +816,7 @@ def test_freeze(env, config, make_repo, users):
     # re-enable forward-port cron after freeze
     _, cron_id = env['ir.model.data'].check_object_reference('forwardport', 'port_forward', context={'active_test': False})
     env['ir.cron'].browse([cron_id]).active = True
-
-    # run crons to process the feedback, run a second time in case of e.g.
-    # forward porting
-    env.run_crons()
-    env.run_crons()
+    env.run_crons('forwardport.port_forward')
 
     assert release_id.state == 'merged'
     assert not env['runbot_merge.pull_requests'].search([
@@ -896,7 +892,7 @@ def test_missing_magic_ref(env, config, make_repo):
     # check that the batch is still here and targeted for the future
     req = env['forwardport.batches'].search([])
     assert len(req) == 1
-    assert req.retry_after > datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
+    assert req.retry_after > datetime.utcnow().isoformat(" ", "seconds")
     # reset retry_after
     req.retry_after = '1900-01-01 01:01:01'
 
@@ -905,7 +901,7 @@ def test_missing_magic_ref(env, config, make_repo):
         [c2] = prod.make_commits(a_head.id, Commit('y', tree={'x': '0'}))
     assert c2 != c
     pr_id.head = c2
-    env.run_crons()
+    env.run_crons(None)
 
     fp_id = env['runbot_merge.pull_requests'].search([('source_id', '=', pr_id.id)])
     assert fp_id
@@ -1161,7 +1157,7 @@ def test_reminder_detached(env, config, make_repo, users):
     pr_c = prod.get_pr(pr_c_id.number)
 
     # region sanity check
-    env.run_crons('forwardport.reminder', 'runbot_merge.feedback_cron', context={'forwardport_updated_before': FAKE_PREV_WEEK})
+    env.run_crons('forwardport.reminder', context={'forwardport_updated_before': FAKE_PREV_WEEK})
 
     assert pr_b.comments == [
         seen(env, pr_b, users),
@@ -1195,7 +1191,7 @@ More info at https://github.com/odoo/odoo/wiki/Mergebot#forward-port
 
     # region check detached
     pr_c_id.write({'parent_id': False, 'detach_reason': 'because'})
-    env.run_crons('forwardport.reminder', 'runbot_merge.feedback_cron', context={'forwardport_updated_before': FAKE_PREV_WEEK})
+    env.run_crons('forwardport.reminder', context={'forwardport_updated_before': FAKE_PREV_WEEK})
 
     assert pr_b.comments[2:] == [
         (users['user'], "@%s @%s child PR %s was modified / updated and has become a normal PR. This PR (and any of its parents) will need to be merged independently as approvals won't cross." % (
diff --git a/runbot_merge/data/merge_cron.xml b/runbot_merge/data/merge_cron.xml
index 3240e3ae..be06078f 100644
--- a/runbot_merge/data/merge_cron.xml
+++ b/runbot_merge/data/merge_cron.xml
@@ -26,8 +26,8 @@
     <field name="model_id" ref="model_runbot_merge_pull_requests_feedback"/>
     <field name="state">code</field>
     <field name="code">model._send()</field>
-    <field name="interval_number">1</field>
-    <field name="interval_type">minutes</field>
+    <field name="interval_number">6</field>
+    <field name="interval_type">hours</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
     <field name="priority">60</field>
@@ -48,8 +48,8 @@
     <field name="model_id" ref="model_runbot_merge_fetch_job"/>
     <field name="state">code</field>
     <field name="code">model._check(True)</field>
-    <field name="interval_number">1</field>
-    <field name="interval_type">minutes</field>
+    <field name="interval_number">6</field>
+    <field name="interval_type">hours</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
     <field name="priority">10</field>
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index a5870cdc..cd331642 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -1774,6 +1774,12 @@ class Feedback(models.Model):
         help="Token field (from repo's project) to use to post messages"
     )
 
+    @api.model_create_multi
+    def create(self, vals_list):
+        # any time a feedback is created, it can be sent
+        self.env.ref('runbot_merge.feedback_cron')._trigger()
+        return super().create(vals_list)
+
     def _send(self):
         ghs = {}
         to_remove = []
@@ -2424,6 +2430,11 @@ class FetchJob(models.Model):
     number = fields.Integer(required=True, group_operator=None)
     closing = fields.Boolean(default=False)
 
+    @api.model_create_multi
+    def create(self, vals_list):
+        self.env.ref('runbot_merge.fetch_prs_cron')._trigger()
+        return super().create(vals_list)
+
     def _check(self, commit=False):
         """
         :param bool commit: commit after each fetch has been executed
diff --git a/runbot_merge/tests/conftest.py b/runbot_merge/tests/conftest.py
index 5f84298c..34d70479 100644
--- a/runbot_merge/tests/conftest.py
+++ b/runbot_merge/tests/conftest.py
@@ -7,16 +7,12 @@ def module():
 @pytest.fixture
 def default_crons():
     return [
-        # env['runbot_merge.project']._check_fetch()
-        'runbot_merge.fetch_prs_cron',
         # env['runbot_merge.project']._check_stagings()
         'runbot_merge.merge_cron',
         # env['runbot_merge.project']._create_stagings()
         'runbot_merge.staging_cron',
         # env['runbot_merge.pull_requests']._check_linked_prs_statuses()
         'runbot_merge.check_linked_prs_status',
-        # env['runbot_merge.pull_requests.feedback']._send()
-        'runbot_merge.feedback_cron',
     ]
 
 @pytest.fixture
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index e738787d..3208e48f 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -3534,7 +3534,6 @@ class TestUnknownPR:
             prx.post_comment('hansen r+', config['role_reviewer']['token'])
         env.run_crons(
             'runbot_merge.fetch_prs_cron',
-            'runbot_merge.feedback_cron',
         )
 
         assert prx.comments == [
@@ -3559,7 +3558,6 @@ class TestUnknownPR:
             prx.post_review('APPROVE', 'hansen r+', config['role_reviewer']['token'])
         env.run_crons(
             'runbot_merge.fetch_prs_cron',
-            'runbot_merge.feedback_cron',
         )
 
         # FIXME: either split out reviews in local or merge reviews & comments in remote

From f367a64481e1c88a5b9cb239e599a44a7c6e3a90 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Wed, 31 Jul 2024 09:40:53 +0200
Subject: [PATCH 179/192] [IMP] *: trigger-ify merge cron

The merge cron is the one in charge of checking staging state and
either integrating the staging into the reference branch (if
successful) or cancelling the staging (if failed).

The most obvious trigger for the merge cron is a change in staging
state from the states computation (transition from pending to either
success or failure). Explicitly cancelling / failing a staging marks
it as inactive so the merge cron isn't actually needed.

However an other major trigger is *timeout*, which doesn't have a
trivial signal. Instead, it needs to be hooked on the `timeout_limit`,
and has to be re-triggered at every update to the `timeout_limit`,
which in normal operations is mostly from "pending" statuses bumping
the timeout limit. In that case, `_trigger` to the `timeout_limit` as
that's where / when we expect a status change.

Worst case scenario with this is we have parasitic wakeups of this
cron, but having half a dozen wakeups unnecessary wakeups in an hour
is still probably better than having a wakeup every minute.
---
 forwardport/tests/conftest.py              |   1 -
 runbot_merge/data/merge_cron.xml           |   4 +-
 runbot_merge/models/pull_requests.py       |  21 ++-
 runbot_merge/tests/conftest.py             |   2 -
 runbot_merge/tests/test_basic.py           | 146 +++++++++++----------
 runbot_merge/tests/test_multirepo.py       |   2 +-
 runbot_merge/tests/test_project_toggles.py |   2 +-
 7 files changed, 99 insertions(+), 79 deletions(-)

diff --git a/forwardport/tests/conftest.py b/forwardport/tests/conftest.py
index 5243d96e..c5951502 100644
--- a/forwardport/tests/conftest.py
+++ b/forwardport/tests/conftest.py
@@ -7,7 +7,6 @@ import requests
 @pytest.fixture
 def default_crons():
     return [
-        'runbot_merge.merge_cron',
         'runbot_merge.staging_cron',
         'runbot_merge.check_linked_prs_status',
     ]
diff --git a/runbot_merge/data/merge_cron.xml b/runbot_merge/data/merge_cron.xml
index be06078f..f6a4f3da 100644
--- a/runbot_merge/data/merge_cron.xml
+++ b/runbot_merge/data/merge_cron.xml
@@ -4,8 +4,8 @@
     <field name="model_id" ref="model_runbot_merge_project"/>
     <field name="state">code</field>
     <field name="code">model._check_stagings(True)</field>
-    <field name="interval_number">1</field>
-    <field name="interval_type">minutes</field>
+    <field name="interval_number">6</field>
+    <field name="interval_type">hours</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
     <field name="priority">30</field>
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index cd331642..0955c853 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -2062,16 +2062,25 @@ class Stagings(models.Model):
                 for context, status in statuses.get(commit.sha, {}).items()
             ]
 
+    def write(self, vals):
+        if timeout := vals.get('timeout_limit'):
+            self.env.ref("runbot_merge.merge_cron")\
+                ._trigger(fields.Datetime.to_datetime(timeout))
+
+        return super().write(vals)
+
     # only depend on staged_at as it should not get modified, but we might
     # update the CI timeout after the staging have been created and we
     # *do not* want to update the staging timeouts in that case
     @api.depends('staged_at')
     def _compute_timeout_limit(self):
+        timeouts = set()
         for st in self:
-            st.timeout_limit = fields.Datetime.to_string(
-                  fields.Datetime.from_string(st.staged_at)
-                + datetime.timedelta(minutes=st.target.project_id.ci_timeout)
-            )
+            t = st.timeout_limit = st.staged_at + datetime.timedelta(minutes=st.target.project_id.ci_timeout)
+            timeouts.add(t)
+        if timeouts:
+            # we might have very different limits for each staging so need to schedule them all
+            self.env.ref("runbot_merge.merge_cron")._trigger_list(timeouts)
 
     @api.depends('batch_ids.prs')
     def _compute_prs(self):
@@ -2147,9 +2156,11 @@ class Stagings(models.Model):
 
             s.state = st
             if s.state != 'pending':
+                self.env.ref("runbot_merge.merge_cron")._trigger()
                 s.staging_end = fields.Datetime.now()
             if update_timeout_limit:
-                s.timeout_limit = fields.Datetime.to_string(datetime.datetime.now() + datetime.timedelta(minutes=s.target.project_id.ci_timeout))
+                s.timeout_limit = datetime.datetime.now() + datetime.timedelta(minutes=s.target.project_id.ci_timeout)
+                self.env.ref("runbot_merge.merge_cron")._trigger(s.timeout_limit)
                 _logger.debug("%s got pending status, bumping timeout to %s (%s)", self, s.timeout_limit, cmap)
 
     def action_cancel(self):
diff --git a/runbot_merge/tests/conftest.py b/runbot_merge/tests/conftest.py
index 34d70479..e200434a 100644
--- a/runbot_merge/tests/conftest.py
+++ b/runbot_merge/tests/conftest.py
@@ -7,8 +7,6 @@ def module():
 @pytest.fixture
 def default_crons():
     return [
-        # env['runbot_merge.project']._check_stagings()
-        'runbot_merge.merge_cron',
         # env['runbot_merge.project']._create_stagings()
         'runbot_merge.staging_cron',
         # env['runbot_merge.pull_requests']._check_linked_prs_statuses()
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 3208e48f..9026aa02 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -3,6 +3,7 @@ import itertools
 import json
 import textwrap
 import time
+from typing import Callable
 from unittest import mock
 
 import pytest
@@ -13,8 +14,9 @@ import odoo
 from utils import _simple_init, seen, matches, get_partner, Commit, pr_page, to_pr, part_of, ensure_one
 
 @pytest.fixture(autouse=True)
-def _configure_statuses(project, repo):
-    project.repo_ids.required_statuses = 'legal/cla,ci/runbot'
+def _configure_statuses(request, project, repo):
+    if 'defaultstatuses' not in request.keywords:
+        project.repo_ids.required_statuses = 'legal/cla,ci/runbot'
 
 @pytest.fixture(autouse=True, params=["statuses", "rpc"])
 def stagings(request, env, project, repo):
@@ -562,19 +564,32 @@ def test_staging_conflict_second(env, repo, users, config):
     assert pr1_id.state == 'error', "now pr1 should be in error"
 
 
-def test_staging_ci_timeout(env, repo, config, page):
+@pytest.mark.defaultstatuses
+@pytest.mark.parametrize('update_op', [
+    pytest.param(
+        lambda _: {'timeout_limit': datetime.datetime.now().isoformat(" ", "seconds")},
+        id="set-timeout-limit",
+    ),
+    pytest.param(
+        lambda timeout: {'staged_at': (datetime.datetime.now() - datetime.timedelta(minutes=2*timeout)).isoformat(" ", "seconds")},
+        id="set-staged-at",
+    ),
+])
+def test_staging_ci_timeout(env, repo, config, page, update_op: Callable[[int], dict]):
     """If a staging timeouts (~ delay since staged greater than
     configured)... requeue?
     """
     with repo:
-        m = repo.make_commit(None, 'initial', None, tree={'f': 'm'})
+        m, _, c2 = repo.make_commits(
+            None,
+            Commit('initial', tree={'f': 'm'}),
+            Commit('first', tree={'f': 'c1'}),
+            Commit('second', tree={'f': 'c2'}),
+        )
         repo.make_ref('heads/master', m)
 
-        c1 = repo.make_commit(m, 'first', None, tree={'f': 'c1'})
-        c2 = repo.make_commit(c1, 'second', None, tree={'f': 'c2'})
         pr = repo.make_pr(title='title', body='body', target='master', head=c2)
-        repo.post_status(pr.head, 'success', 'ci/runbot')
-        repo.post_status(pr.head, 'success', 'legal/cla')
+        repo.post_status(pr.head, 'success')
         pr.post_comment('hansen r+ rebase-merge', config['role_reviewer']['token'])
     env.run_crons()
 
@@ -582,23 +597,26 @@ def test_staging_ci_timeout(env, repo, config, page):
     assert pr_id.staging_id
     timeout = env['runbot_merge.project'].search([]).ci_timeout
 
-    pr_id.staging_id.staged_at = odoo.fields.Datetime.to_string(datetime.datetime.now() - datetime.timedelta(minutes=2*timeout))
-    env.run_crons('runbot_merge.merge_cron', 'runbot_merge.staging_cron')
+    pr_id.staging_id.write(update_op(timeout))
+    env.run_crons('runbot_merge.staging_cron')
     assert pr_id.state == 'error', "timeout should fail the PR"
 
     dangerbox = pr_page(page, pr).cssselect('.alert-danger span')
     assert dangerbox
     assert dangerbox[0].text == 'timed out (>60 minutes)'
 
+@pytest.mark.defaultstatuses
 def test_timeout_bump_on_pending(env, repo, config):
     with repo:
-        m = repo.make_commit(None, 'initial', None, tree={'f': '0'})
+        [m, c] = repo.make_commits(
+            None,
+            Commit('initial', tree={'f': '0'}),
+            Commit('c', tree={'f': '1'}),
+        )
         repo.make_ref('heads/master', m)
 
-        c = repo.make_commit(m, 'c', None, tree={'f': '1'})
         prx = repo.make_pr(title='title', body='body', target='master', head=c)
-        repo.post_status(prx.head, 'success', 'ci/runbot')
-        repo.post_status(prx.head, 'success', 'legal/cla')
+        repo.post_status(prx.head, 'success')
         prx.post_comment('hansen r+', config['role_reviewer']['token'])
     env.run_crons()
 
@@ -606,7 +624,7 @@ def test_timeout_bump_on_pending(env, repo, config):
     old_timeout = odoo.fields.Datetime.to_string(datetime.datetime.now() - datetime.timedelta(days=15))
     st.timeout_limit = old_timeout
     with repo:
-        repo.post_status('staging.master', 'pending', 'ci/runbot')
+        repo.post_status('staging.master', 'pending')
     env.run_crons(None)
     assert st.timeout_limit > old_timeout
 
@@ -1092,63 +1110,63 @@ def test_reopen_merged_pr(env, repo, config, users):
     ]
 
 class TestNoRequiredStatus:
+    @pytest.mark.defaultstatuses
     def test_basic(self, env, repo, config):
         """ check that mergebot can work on a repo with no CI at all
         """
         env['runbot_merge.repository'].search([('name', '=', repo.name)]).status_ids = False
         with repo:
-            m = repo.make_commit(None, 'initial', None, tree={'0': '0'})
+            [m, c] = repo.make_commits(
+                None,
+                Commit('initial', tree={'0': '0'}),
+                Commit('first', tree={'0': '1'}),
+            )
             repo.make_ref('heads/master', m)
 
-            c = repo.make_commit(m, 'first', None, tree={'0': '1'})
-            prx = repo.make_pr(title='title', body='body', target='master', head=c)
-            prx.post_comment('hansen r+', config['role_reviewer']['token'])
+            pr = repo.make_pr(title='title', body='body', target='master', head=c)
+            pr.post_comment('hansen r+', config['role_reviewer']['token'])
         env.run_crons()
 
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number)
-        ])
-        assert pr.state == 'ready'
-        st = pr.staging_id
-        assert st
-        env.run_crons()
+        pr_id = to_pr(env, pr)
+
+        st = env['runbot_merge.stagings'].search([], context={'active_test': False})
         assert st.state == 'success'
-        assert pr.state == 'merged'
+        assert pr_id.state == 'merged'
 
+    @pytest.mark.defaultstatuses
     def test_updated(self, env, repo, config):
         env['runbot_merge.repository'].search([('name', '=', repo.name)]).status_ids = False
         with repo:
-            m = repo.make_commit(None, 'initial', None, tree={'0': '0'})
+            [m, c] = repo.make_commits(
+                None,
+                Commit('initial', tree={'0': '0'}),
+                Commit('first', tree={'0': '1'}),
+            )
             repo.make_ref('heads/master', m)
 
-            c = repo.make_commit(m, 'first', None, tree={'0': '1'})
-            prx = repo.make_pr(title='title', body='body', target='master', head=c)
+            pr = repo.make_pr(title='title', body='body', target='master', head=c)
         env.run_crons()
 
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number)
-        ])
-        assert pr.state == 'validated'
+        pr_id = to_pr(env, pr)
+        assert pr_id.state == 'validated'
 
         # normal push
         with repo:
-            repo.make_commits(c, repo.Commit('second', tree={'0': '2'}), ref=prx.ref)
+            repo.make_commits(c, repo.Commit('second', tree={'0': '2'}), ref=pr.ref)
         env.run_crons()
-        assert pr.state == 'validated'
+        assert pr_id.state == 'validated'
         with repo:
-            prx.post_comment('hansen r+', config['role_reviewer']['token'])
-        assert pr.state == 'ready'
+            pr.post_comment('hansen r+', config['role_reviewer']['token'])
+        assert pr_id.state == 'ready'
 
         # force push
         with repo:
-            repo.make_commits(m, repo.Commit('xxx', tree={'0': 'm'}), ref=prx.ref)
+            repo.make_commits(m, repo.Commit('xxx', tree={'0': 'm'}), ref=pr.ref)
         env.run_crons()
-        assert pr.state == 'validated'
+        assert pr_id.state == 'validated'
         with repo:
-            prx.post_comment('hansen r+', config['role_reviewer']['token'])
-        assert pr.state == 'ready'
+            pr.post_comment('hansen r+', config['role_reviewer']['token'])
+        assert pr_id.state == 'ready'
 
 class TestRetry:
     @pytest.mark.xfail(reason="This may not be a good idea as it could lead to tons of rebuild spam")
@@ -1230,7 +1248,7 @@ class TestRetry:
         with repo:
             pr.post_comment('hansen retry', config['role_' + retrier]['token'])
         assert pr_id.state == 'ready'
-        env.run_crons('runbot_merge.merge_cron', 'runbot_merge.staging_cron')
+        env.run_crons('runbot_merge.staging_cron')
 
         staging_head2 = repo.commit('heads/staging.master')
         assert staging_head2 != staging_head
@@ -1263,7 +1281,7 @@ class TestRetry:
 
         with repo:
             pr.post_comment('hansen retry', config['role_reviewer']['token'])
-        env.run_crons('runbot_merge.merge_cron', 'runbot_merge.staging_cron')
+        env.run_crons('runbot_merge.staging_cron')
 
         with repo:
             repo.post_status('staging.master', 'success', 'legal/cla')
@@ -1293,42 +1311,36 @@ class TestRetry:
             (users['user'], "@{reviewer} retry makes no sense when the PR is not in error.".format_map(users)),
         ]
 
+    @pytest.mark.defaultstatuses
     @pytest.mark.parametrize('disabler', ['user', 'other', 'reviewer'])
     def test_retry_disable(self, env, repo, disabler, users, config):
         with repo:
             prx = _simple_init(repo)
-            repo.post_status(prx.head, 'success', 'ci/runbot')
-            repo.post_status(prx.head, 'success', 'legal/cla')
+            repo.post_status(prx.head, 'success')
             prx.post_comment('hansen r+ delegate=%s rebase-merge' % users['other'],
                              config["role_reviewer"]['token'])
         env.run_crons()
-        assert env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number)
-        ]).staging_id
+        pr_id = to_pr(env, prx)
+        staging_id = pr_id.staging_id
+        assert staging_id
 
-        staging_head = repo.commit('heads/staging.master')
         with repo:
-            repo.post_status('staging.master', 'success', 'legal/cla')
-            repo.post_status('staging.master', 'failure', 'ci/runbot')
+            repo.post_status('staging.master', 'failure')
         env.run_crons()
-        pr = env['runbot_merge.pull_requests'].search([
-            ('repository.name', '=', repo.name),
-            ('number', '=', prx.number)
-        ])
-        assert pr.state == 'error'
+        assert staging_id.state == 'failure'
+        assert not staging_id.active
+        assert pr_id.state == 'error'
 
         with repo:
             prx.post_comment('hansen r-', config['role_' + disabler]['token'])
-        assert pr.state == 'validated'
+        assert pr_id.state == 'validated'
         with repo:
             repo.make_commit(prx.ref, 'third', None, tree={'m': 'c3'})
             # just in case, apparently in some case the first post_status uses the old head...
         with repo:
-            repo.post_status(prx.head, 'success', 'ci/runbot')
-            repo.post_status(prx.head, 'success', 'legal/cla')
+            repo.post_status(prx.head, 'success')
         env.run_crons()
-        assert pr.state == 'validated'
+        assert pr_id.state == 'validated'
 
 class TestMergeMethod:
     """
@@ -1759,7 +1771,7 @@ commits, I need to know how to merge it:
 
             c0 = repo.make_commit(m, 'C0', None, tree={'a': 'b'})
             prx = repo.make_pr(title="gibberish", body="blahblah", target='master', head=c0)
-        env.run_crons('runbot_merge.merge_cron', 'runbot_merge.staging_cron')
+        env.run_crons('runbot_merge.staging_cron')
 
         with repo:
             repo.post_status(prx.head, 'success', 'legal/cla')
@@ -1801,7 +1813,7 @@ commits, I need to know how to merge it:
 
             c0 = repo.make_commit(m, 'C0', None, tree={'a': 'b'})
             prx = repo.make_pr(title="gibberish", body=None, target='master', head=c0)
-        env.run_crons('runbot_merge.merge_cron', 'runbot_merge.staging_cron')
+        env.run_crons('runbot_merge.staging_cron')
 
         with repo:
             repo.post_status(prx.head, 'success', 'legal/cla')
@@ -3077,7 +3089,7 @@ class TestBatching(object):
         with repo:
             repo.post_status('staging.master', 'success', 'ci/runbot')
             repo.post_status('staging.master', 'success', 'legal/cla')
-        env.run_crons('runbot_merge.merge_cron', 'runbot_merge.staging_cron')
+        env.run_crons('runbot_merge.staging_cron')
         assert pr2.state == 'merged'
 
 class TestReviewing:
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 3ac35a8f..3c68bd9e 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -493,7 +493,7 @@ def test_ff_fail(env, project, repo_a, repo_b, config):
     with repo_a, repo_b:
         repo_a.post_status('heads/staging.master', 'success')
         repo_b.post_status('heads/staging.master', 'success')
-    env.run_crons('runbot_merge.merge_cron', 'runbot_merge.staging_cron')
+    env.run_crons('runbot_merge.staging_cron')
     assert repo_b.commit('heads/master').id == cn,\
         "B should still be at the conflicting commit"
     assert repo_a.commit('heads/master').id == root_a,\
diff --git a/runbot_merge/tests/test_project_toggles.py b/runbot_merge/tests/test_project_toggles.py
index 30ce5f78..0dffdccc 100644
--- a/runbot_merge/tests/test_project_toggles.py
+++ b/runbot_merge/tests/test_project_toggles.py
@@ -114,7 +114,7 @@ def test_staging_priority(env, project, repo, config, mode, cutoff, second):
     env[model].browse([cron_id]).write({
         'nextcall': (datetime.datetime.utcnow() + datetime.timedelta(minutes=10)).isoformat(" ", "seconds")
     })
-    env.run_crons('runbot_merge.merge_cron')
+    env.run_crons(None)
     assert not staging.active
     assert not env['runbot_merge.stagings'].search([]).active
     assert env['runbot_merge.split'].search_count([]) == 2

From 3ee3e9cc8140fdf682429e31ee50b04fd7299768 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Thu, 1 Aug 2024 10:15:32 +0200
Subject: [PATCH 180/192] [IMP] *: trigger-ify staging cron

The staging cron turns out to be pretty reasonable to trigger, as we
already have a handler on the transition of a batch to `not blocked`,
which is exactly when we want to create a staging (that and the
completion of the previous staging).

The batch transition is in a compute which is not awesome, but on the
flip side we also cancel active stagings in that exact scenario (if it
applies), so that matches.

The only real finesse is that one of the tests wants to observe the
instant between the end of a staging (and creation of splits) and the
start of the next one, which because the staging cron is triggered by
the failure of the previous staging is now "atomic", requiring
disabling the staging cron, which means the trigger is skipped
entirely. So this requires triggering the staging cron by hand.
---
 forwardport/tests/conftest.py              |  1 -
 runbot_merge/data/merge_cron.xml           |  4 ++--
 runbot_merge/models/batch.py               | 16 +++++++++-------
 runbot_merge/models/pull_requests.py       |  3 +++
 runbot_merge/tests/conftest.py             |  2 --
 runbot_merge/tests/test_basic.py           | 12 ++++++------
 runbot_merge/tests/test_multirepo.py       |  2 +-
 runbot_merge/tests/test_project_toggles.py | 10 ++++++----
 8 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/forwardport/tests/conftest.py b/forwardport/tests/conftest.py
index c5951502..58f57184 100644
--- a/forwardport/tests/conftest.py
+++ b/forwardport/tests/conftest.py
@@ -7,7 +7,6 @@ import requests
 @pytest.fixture
 def default_crons():
     return [
-        'runbot_merge.staging_cron',
         'runbot_merge.check_linked_prs_status',
     ]
 
diff --git a/runbot_merge/data/merge_cron.xml b/runbot_merge/data/merge_cron.xml
index f6a4f3da..aa9de63e 100644
--- a/runbot_merge/data/merge_cron.xml
+++ b/runbot_merge/data/merge_cron.xml
@@ -15,8 +15,8 @@
     <field name="model_id" ref="model_runbot_merge_project"/>
     <field name="state">code</field>
     <field name="code">model._create_stagings(True)</field>
-    <field name="interval_number">1</field>
-    <field name="interval_type">minutes</field>
+    <field name="interval_number">6</field>
+    <field name="interval_type">hours</field>
     <field name="numbercall">-1</field>
     <field name="doall" eval="False"/>
     <field name="priority">40</field>
diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index eace55b4..d920802f 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -223,13 +223,15 @@ class Batch(models.Model):
                     failed and f"{failed} have failed CI",
                 ]))
             else:
-                if batch.blocked and batch.cancel_staging:
-                    if splits := batch.target.split_ids:
-                        splits.unlink()
-                    batch.target.active_staging_id.cancel(
-                        'unstaged by %s becoming ready',
-                        ', '.join(batch.prs.mapped('display_name')),
-                    )
+                if batch.blocked:
+                    self.env.ref("runbot_merge.staging_cron")._trigger()
+                    if batch.cancel_staging:
+                        if splits := batch.target.split_ids:
+                            splits.unlink()
+                        batch.target.active_staging_id.cancel(
+                            'unstaged by %s becoming ready',
+                            ', '.join(batch.prs.mapped('display_name')),
+                        )
                 batch.blocked = False
 
 
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 0955c853..ea55d03b 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -2067,6 +2067,9 @@ class Stagings(models.Model):
             self.env.ref("runbot_merge.merge_cron")\
                 ._trigger(fields.Datetime.to_datetime(timeout))
 
+        if vals.get('active') is False:
+            self.env.ref("runbot_merge.staging_cron")._trigger()
+
         return super().write(vals)
 
     # only depend on staged_at as it should not get modified, but we might
diff --git a/runbot_merge/tests/conftest.py b/runbot_merge/tests/conftest.py
index e200434a..6143afe6 100644
--- a/runbot_merge/tests/conftest.py
+++ b/runbot_merge/tests/conftest.py
@@ -7,8 +7,6 @@ def module():
 @pytest.fixture
 def default_crons():
     return [
-        # env['runbot_merge.project']._create_stagings()
-        'runbot_merge.staging_cron',
         # env['runbot_merge.pull_requests']._check_linked_prs_statuses()
         'runbot_merge.check_linked_prs_status',
     ]
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index 9026aa02..a68d70f8 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -598,7 +598,7 @@ def test_staging_ci_timeout(env, repo, config, page, update_op: Callable[[int],
     timeout = env['runbot_merge.project'].search([]).ci_timeout
 
     pr_id.staging_id.write(update_op(timeout))
-    env.run_crons('runbot_merge.staging_cron')
+    env.run_crons(None)
     assert pr_id.state == 'error', "timeout should fail the PR"
 
     dangerbox = pr_page(page, pr).cssselect('.alert-danger span')
@@ -1248,7 +1248,7 @@ class TestRetry:
         with repo:
             pr.post_comment('hansen retry', config['role_' + retrier]['token'])
         assert pr_id.state == 'ready'
-        env.run_crons('runbot_merge.staging_cron')
+        env.run_crons(None)
 
         staging_head2 = repo.commit('heads/staging.master')
         assert staging_head2 != staging_head
@@ -1281,7 +1281,7 @@ class TestRetry:
 
         with repo:
             pr.post_comment('hansen retry', config['role_reviewer']['token'])
-        env.run_crons('runbot_merge.staging_cron')
+        env.run_crons(None)
 
         with repo:
             repo.post_status('staging.master', 'success', 'legal/cla')
@@ -1771,7 +1771,7 @@ commits, I need to know how to merge it:
 
             c0 = repo.make_commit(m, 'C0', None, tree={'a': 'b'})
             prx = repo.make_pr(title="gibberish", body="blahblah", target='master', head=c0)
-        env.run_crons('runbot_merge.staging_cron')
+        env.run_crons(None)
 
         with repo:
             repo.post_status(prx.head, 'success', 'legal/cla')
@@ -1813,7 +1813,7 @@ commits, I need to know how to merge it:
 
             c0 = repo.make_commit(m, 'C0', None, tree={'a': 'b'})
             prx = repo.make_pr(title="gibberish", body=None, target='master', head=c0)
-        env.run_crons('runbot_merge.staging_cron')
+        env.run_crons(None)
 
         with repo:
             repo.post_status(prx.head, 'success', 'legal/cla')
@@ -3089,7 +3089,7 @@ class TestBatching(object):
         with repo:
             repo.post_status('staging.master', 'success', 'ci/runbot')
             repo.post_status('staging.master', 'success', 'legal/cla')
-        env.run_crons('runbot_merge.staging_cron')
+        env.run_crons(None)
         assert pr2.state == 'merged'
 
 class TestReviewing:
diff --git a/runbot_merge/tests/test_multirepo.py b/runbot_merge/tests/test_multirepo.py
index 3c68bd9e..0efb7cfc 100644
--- a/runbot_merge/tests/test_multirepo.py
+++ b/runbot_merge/tests/test_multirepo.py
@@ -493,7 +493,7 @@ def test_ff_fail(env, project, repo_a, repo_b, config):
     with repo_a, repo_b:
         repo_a.post_status('heads/staging.master', 'success')
         repo_b.post_status('heads/staging.master', 'success')
-    env.run_crons('runbot_merge.staging_cron')
+    env.run_crons(None)
     assert repo_b.commit('heads/master').id == cn,\
         "B should still be at the conflicting commit"
     assert repo_a.commit('heads/master').id == root_a,\
diff --git a/runbot_merge/tests/test_project_toggles.py b/runbot_merge/tests/test_project_toggles.py
index 0dffdccc..db838df6 100644
--- a/runbot_merge/tests/test_project_toggles.py
+++ b/runbot_merge/tests/test_project_toggles.py
@@ -111,15 +111,17 @@ def test_staging_priority(env, project, repo, config, mode, cutoff, second):
     # specifically delay creation of new staging to observe the failed
     # staging's state and the splits
     model, cron_id = env['ir.model.data'].check_object_reference('runbot_merge', 'staging_cron')
-    env[model].browse([cron_id]).write({
-        'nextcall': (datetime.datetime.utcnow() + datetime.timedelta(minutes=10)).isoformat(" ", "seconds")
-    })
+    staging_cron = env[model].browse([cron_id])
+    staging_cron.active = False
+
     env.run_crons(None)
     assert not staging.active
     assert not env['runbot_merge.stagings'].search([]).active
     assert env['runbot_merge.split'].search_count([]) == 2
 
-    env.run_crons()
+    staging_cron.active = True
+    # manually trigger that cron, as having the cron disabled prevented the creation of the triggers entirely
+    env.run_crons('runbot_merge.staging_cron')
 
     # check that st.pr_ids are the PRs we expect
     st = env['runbot_merge.stagings'].search([])

From 157657af49552d8e6b9f4eafc793c980d2d71f91 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 2 Aug 2024 09:13:19 +0200
Subject: [PATCH 181/192] [REM] *: default_crons fixture

With the trigger-ification pretty much complete the only cron that's
still routinely triggered explicitly is the cross-pr check, and it's
that in all modules, so there's no cause to keep an overridable
fixture.
---
 conftest.py                    | 9 ++++-----
 forwardport/tests/conftest.py  | 6 ------
 runbot_merge/tests/conftest.py | 7 -------
 3 files changed, 4 insertions(+), 18 deletions(-)

diff --git a/conftest.py b/conftest.py
index 59f0753e..b043a48b 100644
--- a/conftest.py
+++ b/conftest.py
@@ -540,8 +540,8 @@ def server(request, db, port, module, addons_path, tmpdir):
         p.wait(timeout=30)
 
 @pytest.fixture
-def env(request, port, server, db, default_crons):
-    yield Environment(port, db, default_crons)
+def env(request, port, server, db):
+    yield Environment(port, db)
     if request.node.get_closest_marker('expect_log_errors'):
         if b"Traceback (most recent call last):" not in server[1]:
             pytest.fail("should have found error in logs.")
@@ -1229,11 +1229,10 @@ class LabelsProxy(collections.abc.MutableSet):
         assert r.ok, r.text
 
 class Environment:
-    def __init__(self, port, db, default_crons=()):
+    def __init__(self, port, db):
         self._uid = xmlrpc.client.ServerProxy('http://localhost:{}/xmlrpc/2/common'.format(port)).authenticate(db, 'admin', 'admin', {})
         self._object = xmlrpc.client.ServerProxy('http://localhost:{}/xmlrpc/2/object'.format(port))
         self._db = db
-        self._default_crons = default_crons
 
     def __call__(self, model, method, *args, **kwargs):
         return self._object.execute_kw(
@@ -1255,7 +1254,7 @@ class Environment:
 
 
     def run_crons(self, *xids, **kw):
-        crons = xids or self._default_crons
+        crons = xids or ['runbot_merge.check_linked_prs_status']
         cron_ids = []
         for xid in crons:
             if xid is None:
diff --git a/forwardport/tests/conftest.py b/forwardport/tests/conftest.py
index 58f57184..8764943d 100644
--- a/forwardport/tests/conftest.py
+++ b/forwardport/tests/conftest.py
@@ -4,12 +4,6 @@ import re
 import pytest
 import requests
 
-@pytest.fixture
-def default_crons():
-    return [
-        'runbot_merge.check_linked_prs_status',
-    ]
-
 # public_repo — necessary to leave comments
 # admin:repo_hook — to set up hooks (duh)
 # delete_repo — to cleanup repos created under a user
diff --git a/runbot_merge/tests/conftest.py b/runbot_merge/tests/conftest.py
index 6143afe6..17b1546e 100644
--- a/runbot_merge/tests/conftest.py
+++ b/runbot_merge/tests/conftest.py
@@ -4,13 +4,6 @@ import pytest
 def module():
     return 'runbot_merge'
 
-@pytest.fixture
-def default_crons():
-    return [
-        # env['runbot_merge.pull_requests']._check_linked_prs_statuses()
-        'runbot_merge.check_linked_prs_status',
-    ]
-
 @pytest.fixture
 def project(env, config):
     return env['runbot_merge.project'].create({

From 78cc8835cec024e246e6bfeafbdae9222db8b363 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 5 Aug 2024 08:03:56 +0200
Subject: [PATCH 182/192] [IMP] rubnbot_merge: avoid triggering every cron on
 every test

Since every cron runs on a fresh database, on the first `run_crons`
every single cron in the db will run even though almost none of them
is relevant.

Aside from the slight inefficiency, this creates unnecessary extra
garbage in the test logs.

By setting the `nextcall` of all crons to infinity in the template we
avoid this issue, only triggered crons (or the crons whose nextcall we
set ourselves) will trigger during calls.

This requires adjusting the branch cleanup cron slightly: it didn't
correctly handle the initial run (`lastcall` being false).
---
 conftest.py                                           | 1 +
 runbot_merge/models/crons/cleanup_scratch_branches.py | 9 +++++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/conftest.py b/conftest.py
index b043a48b..b17d2b09 100644
--- a/conftest.py
+++ b/conftest.py
@@ -326,6 +326,7 @@ class DbDict(dict):
             f.write(db)
             f.flush()
             os.fsync(f.fileno())
+            subprocess.run(['psql', db, '-c', "UPDATE ir_cron SET nextcall = 'infinity'"])
 
         return db
 
diff --git a/runbot_merge/models/crons/cleanup_scratch_branches.py b/runbot_merge/models/crons/cleanup_scratch_branches.py
index a5e041ce..cfe05f32 100644
--- a/runbot_merge/models/crons/cleanup_scratch_branches.py
+++ b/runbot_merge/models/crons/cleanup_scratch_branches.py
@@ -9,10 +9,11 @@ class BranchCleanup(models.TransientModel):
     _description = "cleans up scratch refs for deactivated branches"
 
     def _run(self):
-        deactivated = self.env['runbot_merge.branch'].search([
-            ('active', '=', False),
-            ('write_date', '>=', self.env.context['lastcall']),
-        ])
+        domain = [('active', '=', False)]
+        if lastcall := self.env.context['lastcall']:
+            domain.append(('write_date', '>=', lastcall))
+        deactivated = self.env['runbot_merge.branch'].search(domain)
+
         _logger.info(
             "deleting scratch (tmp and staging) refs for branches %s",
             ', '.join(b.name for b in deactivated)

From 8131271a9c10117a98e5972e7ef70aaf056f9bad Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 5 Aug 2024 08:58:05 +0200
Subject: [PATCH 183/192] [FIX] runbot_merge: flaky test

`test_inconsistent_target` was, appropriately, inconsistent, but would
only fail a very small fraction of the time: the issue is that a PR
would switch target between `other` and `master` assuming neither was
an intrinsic blocker *but* the branches are created independently,
just with the same content.

This means if a second ticked over between the creation of the
`master` branch's commit and that of `other`, they would get different
commit hashes (because different timestamp), thus the PR would get 2
commits (or complete nonsense) when targeted to `other`, and the PR
itself would be blocked for lack of a merge method.

The solution is to be slightly less lazy, and create `other` from
`master` instead of copy/pasting the `make_commits` directive. This
means the PR has the exact same number of commits whether targeted to
`master` or `other`, and we now test what we want to test 60 seconds
out of every minute.
---
 runbot_merge/tests/test_batch_consistency.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/runbot_merge/tests/test_batch_consistency.py b/runbot_merge/tests/test_batch_consistency.py
index 9298f143..0cc3b8b0 100644
--- a/runbot_merge/tests/test_batch_consistency.py
+++ b/runbot_merge/tests/test_batch_consistency.py
@@ -106,14 +106,14 @@ def test_inconsistent_target(env, project, make_repo2, users, page, config):
         pr_other = repo1.make_pr(head='something_else', target='master')
 
     with repo2:
-        repo2.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
-        repo2.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/other')
+        [m] = repo2.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
+        repo2.make_ref("heads/other", m)
         repo2.make_commits('master', Commit('b', tree={"b": "b"}), ref='heads/a_pr')
         pr2 = repo2.make_pr(head='a_pr', target='master')
 
     with repo3:
-        repo3.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
-        repo3.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/other')
+        [m] = repo3.make_commits(None, Commit("a", tree={"a": "a"}), ref='heads/master')
+        repo3.make_ref("heads/other", m)
         repo3.make_commits('master', Commit('b', tree={"b": "b"}), ref='heads/a_pr')
         pr3 = repo3.make_pr(head='a_pr', target='master')
 
@@ -171,7 +171,7 @@ Inconsistent targets:
     env.run_crons()
     assert not pr1_id.blocked
     assert not pr2_id.blocked
-    assert b.blocked
+    assert b.blocked == "Multiple target branches: 'other, master'"
     assert env['runbot_merge.stagings'].search_count([]) == 0
 
     act = pr2_id.button_split()

From ec01523875d93284a0af4e0ed87c00e6eea73a45 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 5 Aug 2024 09:03:39 +0200
Subject: [PATCH 184/192] [IMP] runbot_merge: prune repo during maintenance

The weekly maintenance would not prune refs. This is not an issue on
odoo/odoo because development branches are in a separate repository,
thus never fetched (we push to them but only using local commits and
remote refs).

However on repos like odoo/documentation the reference and development
branches are collocated, the lack of pruning thus keeps every
development branch alive locally, even years after the branch has been
deleted in the repository.

By pruning remote-tracking refs before GC-ing, we should have cleaner
local clones, and better packing.
---
 runbot_merge/models/crons/git_maintenance.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/runbot_merge/models/crons/git_maintenance.py b/runbot_merge/models/crons/git_maintenance.py
index 29dd028a..3c34adfa 100644
--- a/runbot_merge/models/crons/git_maintenance.py
+++ b/runbot_merge/models/crons/git_maintenance.py
@@ -29,9 +29,16 @@ class GC(models.TransientModel):
                 continue
 
             _gc.info('Running maintenance on %s', repo.name)
+            r = repo_git\
+                .stdout(True)\
+                .with_config(stderr=subprocess.STDOUT, text=True, check=False)\
+                .remote('prune', 'origin')
+            if r.returncode:
+                _gc.warning("Prune failure (status=%d):\n%s", r.returncode, r.stdout)
+
             r = repo_git\
                 .stdout(True)\
                 .with_config(stderr=subprocess.STDOUT, text=True, check=False)\
                 .gc('--prune=now', aggressive=True)
             if r.returncode:
-                _gc.warning("Maintenance failure (status=%d):\n%s", r.returncode, r.stdout)
+                _gc.warning("GC failure (status=%d):\n%s", r.returncode, r.stdout)

From ff6f046811ce00eec9472df842f3e8d913a40aaf Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 5 Aug 2024 09:09:23 +0200
Subject: [PATCH 185/192] [CHG] runbot_merge: deskill mergebot feedback

twas heehee in the moment but it's not really a long term hoohoo
haahaa.
---
 runbot_merge/models/commands.py      | 2 +-
 runbot_merge/models/pull_requests.py | 2 +-
 runbot_merge/tests/test_basic.py     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/runbot_merge/models/commands.py b/runbot_merge/models/commands.py
index 675d71eb..fdee31c6 100644
--- a/runbot_merge/models/commands.py
+++ b/runbot_merge/models/commands.py
@@ -269,7 +269,7 @@ class Parser:
             if handler:
                 yield handler(self)
             elif '!' in token:
-                raise CommandError("skill issue, noob")
+                raise CommandError("no need to scream")
             else:
                 raise CommandError(f"unknown command {token!r}")
 
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index ea55d03b..2901def0 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -947,7 +947,7 @@ For your own safety I've ignored *everything in your entire comment*.
                     msg = "you can't set a forward-port limit."
                 # NO!
                 case _:
-                    msg = f"you can't {command}. Skill issue."
+                    msg = f"you can't {command}."
             if msg is not None:
                 rejections.append(msg)
 
diff --git a/runbot_merge/tests/test_basic.py b/runbot_merge/tests/test_basic.py
index a68d70f8..18d75d73 100644
--- a/runbot_merge/tests/test_basic.py
+++ b/runbot_merge/tests/test_basic.py
@@ -3158,7 +3158,7 @@ class TestReviewing:
         assert prx.comments == [
             (users['reviewer'], 'hansen r+'),
             seen(env, prx, users),
-            (users['user'], "@{} you can't review+. Skill issue.".format(users['reviewer'])),
+            (users['user'], "@{} you can't review+.".format(users['reviewer'])),
         ]
 
     def test_self_review_success(self, env, repo, users, config):

From 52f2b1e381f583f23cade1abb2650bb348375227 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 5 Aug 2024 15:18:19 +0200
Subject: [PATCH 186/192] [FIX] runbot_merge: don't show details of merged PR
 on GH

Detailed statuses are useful in the actual PR dashboard as that allows
direct access to the builds, however in the PR where it's only a
picture it's useless, so fold that information. Also fold it when a PR
is staged.

And while at it add a note / sub-title that the PR is staged.

Fixes #919
---
 runbot_merge/controllers/dashboard.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/runbot_merge/controllers/dashboard.py b/runbot_merge/controllers/dashboard.py
index 2db8def1..9cdc55d8 100644
--- a/runbot_merge/controllers/dashboard.py
+++ b/runbot_merge/controllers/dashboard.py
@@ -355,7 +355,14 @@ def render_full_table(pr, branches, repos, batches):
                 for p in ps['prs']:
                     pr = p['pr']
                     attached = attached and p['attached']
-                    sub = ": staging failed" if pr.error else ""
+
+                    if pr.staging_id:
+                        sub = ": is staged"
+                    elif pr.error:
+                        sub = ": staging failed"
+                    else:
+                        sub = ""
+
                     lines = [
                         Line([Text(
                             f"#{p['number']}{sub}",
@@ -366,7 +373,7 @@ def render_full_table(pr, branches, repos, batches):
                     ]
 
                     # no need for details if closed or in error
-                    if not (p['closed'] or pr.error):
+                    if pr.state not in ('merged', 'closed', 'error') and not pr.staging_id:
                         if pr.draft:
                             lines.append(Line([boxes[False], Text("is in draft", font, error)]))
                         lines.extend([

From 0334225114fd650cfaab54a96125f6f024be0326 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 5 Aug 2024 15:57:06 +0200
Subject: [PATCH 187/192] [FIX] runbot_merge: read sha first in commit statuses
 broadcast

Because the first operation the notification task performs is updating
the commit, this has to be flushed in order to read-back the commit
hash (probably fixed in later version).

This means if updating the flag triggers a serialization
failure (which happens and is normal) we transition to the `exception`
handler, which *tries to retrieve the sha again* and we get an "in
failed transaction" error on top of the current "serialization
failure", leading to a giant traceback.

Also an unhelpful one since a serialization failure is expected in
this cron.

- Move the reads with no write dependency out of the `try`, there's no
  reason for them to fail, and notably memoize the `sha`.
- Split the handling of serialization failures out of the normal one,
  and just log an `info` (we could actually log nothing, probably).
- Also set the precommit data just before the commit, in case staging
  tracking is ever a thing which could use it.

Fixes #909
---
 runbot_merge/models/pull_requests.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index 2901def0..c2b243bb 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -13,7 +13,7 @@ from functools import reduce
 from operator import itemgetter
 from typing import Optional, Union, List, Iterator, Tuple
 
-import psycopg2
+import psycopg2.errors
 import sentry_sdk
 import werkzeug
 from markupsafe import Markup
@@ -1934,25 +1934,30 @@ class Commit(models.Model):
         PRs = self.env['runbot_merge.pull_requests']
         # chances are low that we'll have more than one commit
         for c in self.search([('to_check', '=', True)]):
+            sha = c.sha
+            pr = PRs.search([('head', '=', sha)])
+            stagings = Stagings.search([
+                ('head_ids.sha', '=', sha),
+                ('state', '=', 'pending'),
+                ('target.project_id.staging_statuses', '=', True),
+            ])
             try:
                 c.to_check = False
-                pr = PRs.search([('head', '=', c.sha)])
+                self.flush(['to_check'], c)
                 if pr:
-                    self.env.cr.precommit.data['change-message'] =\
-                        f"statuses changed on {c.sha}"
                     pr._validate(c.statuses)
 
-                stagings = Stagings.search([
-                    ('head_ids.sha', '=', c.sha),
-                    ('state', '=', 'pending'),
-                    ('target.project_id.staging_statuses', '=', True),
-                ])
                 if stagings:
                     stagings._notify(c)
+            except psycopg2.errors.SerializationFailure:
+                _logger.info("Failed to apply commit %s (%s): serialization failure", c, sha)
+                self.env.cr.rollback()
             except Exception:
-                _logger.exception("Failed to apply commit %s (%s)", c, c.sha)
+                _logger.exception("Failed to apply commit %s (%s)", c, sha)
                 self.env.cr.rollback()
             else:
+                self.env.cr.precommit.data['change-message'] = \
+                    f"statuses changed on {sha}"
                 self.env.cr.commit()
 
     _sql_constraints = [

From 5dcdfb113831b5c9724453d663720bb6df6f1678 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 9 Aug 2024 10:05:42 +0200
Subject: [PATCH 188/192] [IMP] testing: avoid error on test failure / abort

It's not entirely clear but I assume if / when a test fails or is
aborted (via C-c) pytest kills the stdout somehow. Either way this
causes a bunch of `OSError` to be logged out as we try to write to
closed pipes.

If that occurs, assume the test is gone and just bail out of the
thread.
---
 conftest.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/conftest.py b/conftest.py
index 1357fdf3..83d24896 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import datetime
+import errno
 import select
 import shutil
 import threading
@@ -515,7 +516,12 @@ def server(request, db, port, module, addons_path, tmpdir):
                 r = os.read(inpt, 4096)
                 if not r:
                     break
-                os.write(output, r)
+                try:
+                    os.write(output, r)
+                except OSError as e:
+                    if e.errno == errno.EBADF:
+                        break
+                    raise
                 buf.extend(r)
         os.close(inpt)
 

From a567960d52063b0d8896fb9beb2dee905673ff28 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 9 Aug 2024 10:08:36 +0200
Subject: [PATCH 189/192] [FIX] runbot_merge: 16.0 orm warning

Regular indexing doesn't work if unaccent is enabled, and the ORM
warns about it.
---
 runbot_merge/models/batch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runbot_merge/models/batch.py b/runbot_merge/models/batch.py
index d920802f..df52fc38 100644
--- a/runbot_merge/models/batch.py
+++ b/runbot_merge/models/batch.py
@@ -99,7 +99,7 @@ class Batch(models.Model):
 
     # unlike on PRs, this does not get detached... ? (because batches can be
     # partially detached so that's a PR-level concern)
-    parent_path = fields.Char(index=True)
+    parent_path = fields.Char(index=True, unaccent=False)
     parent_id = fields.Many2one("runbot_merge.batch")
     genealogy_ids = fields.Many2many(
         "runbot_merge.batch",

From f378689a0db48a69554af86152ea66c17b9748d4 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 9 Aug 2024 10:09:20 +0200
Subject: [PATCH 190/192] [FIX] runbot_merge: dumbshit 16.0 view validation
 complaints

---
 runbot_merge/views/batch.xml                |  6 +++---
 runbot_merge/views/mergebot.xml             | 24 ++++++++++-----------
 runbot_merge/views/runbot_merge_project.xml |  4 ++--
 3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/runbot_merge/views/batch.xml b/runbot_merge/views/batch.xml
index ed3f04ba..0a62c0cf 100644
--- a/runbot_merge/views/batch.xml
+++ b/runbot_merge/views/batch.xml
@@ -59,7 +59,7 @@
                         <group colspan="4">
                             <field name="all_prs" nolabel="1" readonly="1">
                                 <tree>
-                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link" title="open PR"/>
                                     <field name="display_name"/>
                                     <field name="repository"/>
                                     <field name="state"/>
@@ -71,7 +71,7 @@
                         <group colspan="4">
                             <field name="genealogy_ids" nolabel="1" readonly="1">
                                 <tree decoration-muted="id == parent.id">
-                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link" title="open batch"/>
                                     <field name="name"/>
                                     <field name="target"/>
                                     <field name="all_prs" widget="many2many_tags"/>
@@ -83,7 +83,7 @@
                         <group colspan="4">
                             <field name="staging_ids" nolabel="1" readonly="1">
                                 <tree>
-                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link" title="open staging"/>
                                     <field name="staged_at"/>
                                     <field name="state"/>
                                     <field name="reason"/>
diff --git a/runbot_merge/views/mergebot.xml b/runbot_merge/views/mergebot.xml
index 66888469..2b13e6eb 100644
--- a/runbot_merge/views/mergebot.xml
+++ b/runbot_merge/views/mergebot.xml
@@ -20,7 +20,7 @@
                     <separator string="Required Statuses"/>
                     <field name="status_ids">
                         <tree editable="bottom">
-                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                            <button type="object" name="get_formview_action" icon="fa-external-link" title="open status"/>
                             <field name="context"/>
                             <field name="branch_filter"/>
                             <field name="prs"/>
@@ -56,7 +56,7 @@
                     </group>
                     <field name="staging_ids" nolabel="1" readonly="1">
                         <tree default_order="staged_at desc">
-                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                            <button type="object" name="get_formview_action" icon="fa-external-link" title="open staging"/>
                             <field name="id"/>
                             <field name="staged_at"/>
                             <field name="state"/>
@@ -202,7 +202,7 @@
                                 <group colspan="4">
                                     <field name="delegates" nolabel="1">
                                         <tree>
-                                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                            <button type="object" name="get_formview_action" icon="fa-external-link" title="open partner"/>
                                             <field name="name"/>
                                             <field name="github_login"/>
                                         </tree>
@@ -215,7 +215,7 @@
                                 <group colspan="4">
                                     <field name="staging_ids" nolabel="1" readonly="1">
                                         <tree>
-                                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                            <button type="object" name="get_formview_action" icon="fa-external-link" title="open staging"/>
                                             <field name="staged_at"/>
                                             <field name="state"/>
                                             <field name="reason"/>
@@ -239,7 +239,7 @@
                               <group colspan="4">
                                 <field name="forwardport_ids" nolabel="1" readonly="True">
                                   <tree>
-                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link" title="open PR"/>
                                     <field name="target" string="Branch"/>
                                     <field name="number"/>
                                   </tree>
@@ -337,7 +337,7 @@
                         <group string="Heads">
                             <field name="head_ids" colspan="2" nolabel="1" readonly="1">
                                 <tree>
-                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link" title="open head"/>
                                     <field name="sha"/>
                                 </tree>
                             </field>
@@ -345,7 +345,7 @@
                         <group string="Commits">
                             <field name="commit_ids" colspan="2" nolabel="1" readonly="1">
                                 <tree>
-                                    <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                    <button type="object" name="get_formview_action" icon="fa-external-link" title="open commit"/>
                                     <field name="sha"/>
                                 </tree>
                             </field>
@@ -354,7 +354,7 @@
                     <group string="Batches">
                         <field name="batch_ids" colspan="4" nolabel="1" readonly="1">
                             <tree>
-                                <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                <button type="object" name="get_formview_action" icon="fa-external-link" title="open batch"/>
                                 <field name="name"/>
                                 <field name="prs" widget="many2many_tags"/>
                             </tree>
@@ -363,7 +363,7 @@
                     <group string="PRs">
                         <field name="pr_ids" colspan="4" nolabel="1" readonly="1">
                             <tree>
-                                <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                                <button type="object" name="get_formview_action" icon="fa-external-link" title="open pr"/>
                                 <field name="display_name"/>
                                 <field name="github_url" widget="url"/>
                                 <field name="url" widget="url"/>
@@ -402,7 +402,7 @@
                     <separator string="Pull Requests"/>
                     <field name="pull_requests" nolabel="1">
                         <tree>
-                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                            <button type="object" name="get_formview_action" icon="fa-external-link" title="open pr"/>
                             <field name="display_name"/>
                             <field name="state"/>
                         </tree>
@@ -410,7 +410,7 @@
                     <separator string="Stagings (commits)"/>
                     <field name="commit_ids" nolabel="1" readonly="1">
                         <tree>
-                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                            <button type="object" name="get_formview_action" icon="fa-external-link" title="open commit"/>
                             <field name="target"/>
                             <field name="id"/>
                             <field name="staged_at"/>
@@ -420,7 +420,7 @@
                     <separator string="Stagings (heads)"/>
                     <field name="head_ids" nolabel="1" readonly="1">
                         <tree>
-                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                            <button type="object" name="get_formview_action" icon="fa-external-link" title="open head"/>
                             <field name="target"/>
                             <field name="id"/>
                             <field name="staged_at"/>
diff --git a/runbot_merge/views/runbot_merge_project.xml b/runbot_merge/views/runbot_merge_project.xml
index 6e83376f..626cef9b 100644
--- a/runbot_merge/views/runbot_merge_project.xml
+++ b/runbot_merge/views/runbot_merge_project.xml
@@ -62,7 +62,7 @@
                     <field name="repo_ids">
                         <tree>
                             <field name="sequence" widget="handle"/>
-                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                            <button type="object" name="get_formview_action" icon="fa-external-link" title="open repo"/>
                             <field name="name"/>
                             <field name="branch_filter"/>
                             <field name="status_ids" widget="many2many_tags"/>
@@ -72,7 +72,7 @@
                     <field name="branch_ids">
                         <tree editable="bottom" decoration-muted="not active">
                             <field name="sequence" widget="handle" />
-                            <button type="object" name="get_formview_action" icon="fa-external-link"/>
+                            <button type="object" name="get_formview_action" icon="fa-external-link" title="open branch"/>
                             <field name="name"/>
                             <field name="active" widget="boolean_toggle"/>
                             <field name="staging_enabled" widget="boolean_toggle"/>

From 61b92b22245f9323b55a83f006feb38b1e9b78c7 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 9 Aug 2024 10:10:01 +0200
Subject: [PATCH 191/192] [IMP] runbot_merge: use native support for tracking
 messages

odoo/odoo@1341d52 added native support for tracking / overriding
tracking message so we don't need `change-message `anymore. The calls
had to be modified a bit as `_track_set_log_message` has to be invoked
on the records for which the tracking message is being set / updated.

Sadly the same for authors was only added in 16.3 *and* it's
unsuitable for our needs as we want to set the author without knowing
the scope of affected records (at least in the controller).

That way hopefully in 17.0 we can remove the `_message_compute_author`
override and things should work out. And maybe for 19.0 we can get the
ability to set a per-model (or even global) fallback author into
standard.
---
 runbot_merge/controllers/__init__.py |  2 +-
 runbot_merge/models/__init__.py      |  1 +
 runbot_merge/models/mail_thread.py   | 32 ++++++++++++++++++++++++++++
 runbot_merge/models/pull_requests.py | 32 ++++------------------------
 4 files changed, 38 insertions(+), 29 deletions(-)
 create mode 100644 runbot_merge/models/mail_thread.py

diff --git a/runbot_merge/controllers/__init__.py b/runbot_merge/controllers/__init__.py
index 8b18a846..b25e9a55 100644
--- a/runbot_merge/controllers/__init__.py
+++ b/runbot_merge/controllers/__init__.py
@@ -258,7 +258,7 @@ def handle_pr(env, event):
     sender = env['res.partner'].search([('github_login', '=', event['sender']['login'])], limit=1)
     if not sender:
         sender = env['res.partner'].create({'name': event['sender']['login'], 'github_login': event['sender']['login']})
-    env.cr.precommit.data['change-author'] = sender.id
+    env['runbot_merge.pull_requests']._track_set_author(sender.id, fallback=True)
 
     if event['action'] == 'opened':
         author_name = pr['user']['login']
diff --git a/runbot_merge/models/__init__.py b/runbot_merge/models/__init__.py
index f15f98f2..ac19320c 100644
--- a/runbot_merge/models/__init__.py
+++ b/runbot_merge/models/__init__.py
@@ -1,3 +1,4 @@
+from . import mail_thread
 from . import ir_actions
 from . import res_partner
 from . import project
diff --git a/runbot_merge/models/mail_thread.py b/runbot_merge/models/mail_thread.py
new file mode 100644
index 00000000..d788e49f
--- /dev/null
+++ b/runbot_merge/models/mail_thread.py
@@ -0,0 +1,32 @@
+from collections import ChainMap
+
+from odoo import models
+from odoo.tools import ConstantMapping
+
+
+class MailThread(models.AbstractModel):
+    _inherit = 'mail.thread'
+
+    def _track_set_author(self, author, *, fallback=False):
+        """ Set the author of the tracking message. """
+        if not self._track_get_fields():
+            return
+        authors = self.env.cr.precommit.data.setdefault(f'mail.tracking.author.{self._name}', {})
+        if fallback:
+            details = authors
+            if isinstance(authors, ChainMap):
+                details = authors.maps[0]
+            self.env.cr.precommit.data[f'mail.tracking.author.{self._name}'] = ChainMap(
+                details,
+                ConstantMapping(author),
+            )
+        else:
+            for id_ in self.ids:
+                authors[id_] = author
+
+    def _message_compute_author(self, author_id=None, email_from=None, raise_on_email=True):
+        if author_id is None and self and 'id' in self:
+            t = self.env.cr.precommit.data.get(f'mail.tracking.author.{self._name}', {})
+            if len(authors := {t.get(r.id) for r in self}) == 1:
+                author_id = authors.pop()
+        return super()._message_compute_author(author_id, email_from, raise_on_email)
diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py
index c2b243bb..b39b6681 100644
--- a/runbot_merge/models/pull_requests.py
+++ b/runbot_merge/models/pull_requests.py
@@ -954,7 +954,7 @@ For your own safety I've ignored *everything in your entire comment*.
         cmdstr = ', '.join(map(str, cmds))
         if not rejections:
             _logger.info("%s (%s) applied %s", login, name, cmdstr)
-            self.env.cr.precommit.data['change-author'] = author.id
+            self._track_set_author(author.id, fallback=True)
             return 'applied ' + cmdstr
 
         self.env.cr.rollback()
@@ -1121,27 +1121,6 @@ For your own safety I've ignored *everything in your entire comment*.
             self.env.ref('runbot_merge.check_linked_prs_status')._trigger()
         return None
 
-    def message_post(self, **kw):
-        if author := self.env.cr.precommit.data.get('change-author'):
-            kw['author_id'] = author
-        if message := self.env.cr.precommit.data.get('change-message'):
-            kw['body'] = html_escape(message)
-        return super().message_post(**kw)
-
-    def _message_log(self, **kw):
-        if author := self.env.cr.precommit.data.get('change-author'):
-            kw['author_id'] = author
-        if message := self.env.cr.precommit.data.get('change-message'):
-            kw['body'] = html_escape(message)
-        return super()._message_log(**kw)
-
-    def _message_log_batch(self, **kw):
-        if author := self.env.cr.precommit.data.get('change-author'):
-            kw['author_id'] = author
-        if message := self.env.cr.precommit.data.get('change-message'):
-            kw['bodies'] = {p.id: html_escape(message) for p in self}
-        return super()._message_log_batch(**kw)
-
     def _pr_acl(self, user):
         if not self:
             return ACL(False, False, False)
@@ -1946,6 +1925,7 @@ class Commit(models.Model):
                 self.flush(['to_check'], c)
                 if pr:
                     pr._validate(c.statuses)
+                    pr._track_set_log_message(html_escape(f"statuses changed on {sha}"))
 
                 if stagings:
                     stagings._notify(c)
@@ -1956,8 +1936,6 @@ class Commit(models.Model):
                 _logger.exception("Failed to apply commit %s (%s)", c, sha)
                 self.env.cr.rollback()
             else:
-                self.env.cr.precommit.data['change-message'] = \
-                    f"statuses changed on {sha}"
                 self.env.cr.commit()
 
     _sql_constraints = [
@@ -2199,9 +2177,8 @@ class Stagings(models.Model):
 
     def fail(self, message, prs=None):
         _logger.info("Staging %s failed: %s", self, message)
-        self.env.cr.precommit.data['change-message'] =\
-            f'staging {self.id} failed: {message}'
         prs = prs or self.batch_ids.prs
+        prs._track_set_log_message(f'staging {self.id} failed: {message}')
         prs.error = True
         for pr in prs:
            self.env.ref('runbot_merge.pr.staging.fail')._send(
@@ -2317,9 +2294,8 @@ class Stagings(models.Model):
                     'reason': str(e.__cause__ or e.__context__ or e)
                 })
             else:
-                self.env.cr.precommit.data['change-message'] =\
-                    f'staging {self.id} succeeded'
                 prs = self.mapped('batch_ids.prs')
+                prs._track_set_log_message(f'staging {self.id} succeeded')
                 logger.info(
                     "%s FF successful, marking %s as merged",
                     self, prs

From d7a78f89d0877e1688f999e1d188fc519e5049d5 Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Fri, 9 Aug 2024 12:51:00 +0200
Subject: [PATCH 192/192] [FIX] conftest: pytest warning

Forgot to declare `defaultstatuses` when it was introduced in
f367a64481e1c88a5b9cb239e599a44a7c6e3a90. pytest righteously warns
when markers are not declated, so do that (turns out running pytest
with `-We` can be useful, though it also requires `-W
ignore::DeprecationWarning::importlib._boostrap` because reportlab
does weird shit).
---
 conftest.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/conftest.py b/conftest.py
index 83d24896..8a770095 100644
--- a/conftest.py
+++ b/conftest.py
@@ -87,7 +87,7 @@ def pytest_addoption(parser):
     parser.addoption('--coverage', action='store_true')
 
     parser.addoption(
-        '--tunnel', action="store", type="choice", choices=['', 'ngrok', 'localtunnel'], default='',
+        '--tunnel', action="store", choices=['', 'ngrok', 'localtunnel'], default='',
         help="Which tunneling method to use to expose the local Odoo server "
              "to hook up github's webhook. ngrok is more reliable, but "
              "creating a free account is necessary to avoid rate-limiting "
@@ -105,6 +105,10 @@ def pytest_configure(config):
         "markers",
         "expect_log_errors(reason): allow and require tracebacks in the log",
     )
+    config.addinivalue_line(
+        "markers",
+        "defaultstatuses: use the statuses `default` rather than `ci/runbot,legal/cla`",
+    )
 
 def pytest_unconfigure(config):
     if not is_manager(config):