2024-10-02 17:14:09 +07:00
|
|
|
""" Implements direct (unstaged) patching.
|
|
|
|
|
|
|
|
Useful for massive data changes which are a pain to merge normally but very
|
|
|
|
unlikely to break things (e.g. i18n), fixes so urgent staging is an unacceptable
|
|
|
|
overhead, or FBI backdoors oh wait forget about that last one.
|
|
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
|
[FIX] runbot_merge: handle missing patch commits
Commits can take some time to propagate through the network (I guess),
or human error can lead to the wrong commit being set.
Either way, because the entire thing was done using a single fetch in
`check=True` the cron job would fail entirely if any of the patch
commits was yet unavailable.
Update the updater to:
- fallback on individual fetches
- remove the patch from the set of applicable patch if we (still)
can't find its commit
I'd have hoped `fetch` could retrieve whatever it found, but
apparently the server just crashes out when it doesn't find the commit
we ask for, and `fetch` doesn't update anything.
No linked issue because I apparently forgot to jot it down (and only
remembered about this issue with the #1063 patching issue) but this
was reported by mat last week (2025-02-21) when they were wondering
why one of their patches was taking a while:
- At 0832 patch was created by automated script.
- At 0947, an attempt to apply was made, the commit was not found.
- At 1126, a second attempt was made but an other patch had been
created whose commit was not found, failing both.
- At 1255, there was a concurrency error ("cannot lock ref" on the
target branch).
- Finally at 1427 the patch was applied.
All in all it took 6 hours to apply the patch, which is 3-4 staging
cycles.
2025-02-25 20:00:45 +07:00
|
|
|
import collections
|
2025-01-29 19:29:53 +07:00
|
|
|
import contextlib
|
2024-10-02 17:14:09 +07:00
|
|
|
import logging
|
|
|
|
import pathlib
|
|
|
|
import re
|
|
|
|
import subprocess
|
|
|
|
import tarfile
|
|
|
|
import tempfile
|
|
|
|
from dataclasses import dataclass
|
[FIX] runbot_merge: handle missing patch commits
Commits can take some time to propagate through the network (I guess),
or human error can lead to the wrong commit being set.
Either way, because the entire thing was done using a single fetch in
`check=True` the cron job would fail entirely if any of the patch
commits was yet unavailable.
Update the updater to:
- fallback on individual fetches
- remove the patch from the set of applicable patch if we (still)
can't find its commit
I'd have hoped `fetch` could retrieve whatever it found, but
apparently the server just crashes out when it doesn't find the commit
we ask for, and `fetch` doesn't update anything.
No linked issue because I apparently forgot to jot it down (and only
remembered about this issue with the #1063 patching issue) but this
was reported by mat last week (2025-02-21) when they were wondering
why one of their patches was taking a while:
- At 0832 patch was created by automated script.
- At 0947, an attempt to apply was made, the commit was not found.
- At 1126, a second attempt was made but an other patch had been
created whose commit was not found, failing both.
- At 1255, there was a concurrency error ("cannot lock ref" on the
target branch).
- Finally at 1427 the patch was applied.
All in all it took 6 hours to apply the patch, which is 3-4 staging
cycles.
2025-02-25 20:00:45 +07:00
|
|
|
from datetime import timedelta
|
2024-10-02 17:14:09 +07:00
|
|
|
from email import message_from_string, policy
|
|
|
|
from email.utils import parseaddr
|
|
|
|
from typing import Union
|
|
|
|
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
from markupsafe import Markup
|
|
|
|
|
2025-02-27 20:43:50 +07:00
|
|
|
from odoo import models, fields, api
|
2024-10-02 17:14:09 +07:00
|
|
|
from odoo.exceptions import ValidationError
|
|
|
|
from odoo.tools.mail import plaintext2html
|
|
|
|
|
|
|
|
from .pull_requests import Branch
|
|
|
|
from .. import git
|
|
|
|
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
FILE_PATTERN = re.compile(r"""
|
|
|
|
# paths with spaces don't work well as the path can be followed by a timestamp
|
|
|
|
# (in an unspecified format?)
|
2024-11-18 18:37:44 +07:00
|
|
|
---\x20(?P<prefix_a>a/)?(?P<file_from>\S+)(:?\s.*)?\n
|
|
|
|
\+\+\+\x20(?P<prefix_b>b/)?(?P<file_to>\S+)(:?\s.*)?\n
|
2024-10-02 17:14:09 +07:00
|
|
|
@@\x20-(\d+(,\d+)?)\x20\+(\d+(,\d+)?)\x20@@ # trailing garbage
|
|
|
|
""", re.VERBOSE)
|
|
|
|
|
|
|
|
|
|
|
|
Authorship = Union[None, tuple[str, str], tuple[str, str, str]]
|
|
|
|
@dataclass
|
|
|
|
class ParseResult:
|
|
|
|
kind: str
|
|
|
|
author: Authorship
|
|
|
|
committer: Authorship
|
|
|
|
message: str
|
|
|
|
patch: str
|
|
|
|
|
|
|
|
|
|
|
|
def expect(line: str, starts_with: str, message: str) -> str:
|
|
|
|
if not line.startswith(starts_with):
|
|
|
|
raise ValidationError(message)
|
|
|
|
return line
|
|
|
|
|
|
|
|
|
|
|
|
def parse_show(p: Patch) -> ParseResult:
|
|
|
|
# headers are Author, Date or Author, AuthorDate, Commit, CommitDate
|
|
|
|
# commit message is indented 4 spaces
|
2024-11-28 19:46:44 +07:00
|
|
|
lines = (l + '\n' for l in p.patch.splitlines(keepends=False))
|
|
|
|
if not next(lines, '').startswith("commit "):
|
2024-10-02 17:14:09 +07:00
|
|
|
raise ValidationError("Invalid patch")
|
|
|
|
name, email = parseaddr(
|
2024-11-28 19:46:44 +07:00
|
|
|
expect(next(lines, ''), "Author:", "Missing author")
|
2024-10-02 17:14:09 +07:00
|
|
|
.split(maxsplit=1)[1])
|
2024-11-28 19:46:44 +07:00
|
|
|
date: str = next(lines, '')
|
2024-10-02 17:14:09 +07:00
|
|
|
header, date = date.split(maxsplit=1)
|
|
|
|
author = (name, email, date)
|
|
|
|
if header.startswith("Date:"):
|
|
|
|
committer = author
|
|
|
|
elif header.startswith("AuthorDate:"):
|
2024-11-28 19:46:44 +07:00
|
|
|
commit = expect(next(lines, ''), "Commit:", "Missing committer")
|
|
|
|
commit_date = expect(next(lines, ''), "CommitDate:", "Missing commit date")
|
2024-10-02 17:14:09 +07:00
|
|
|
name, email = parseaddr(commit.split(maxsplit=1)[1])
|
|
|
|
committer = (name, email, commit_date.split(maxsplit=1)[1])
|
|
|
|
else:
|
|
|
|
raise ValidationError(
|
|
|
|
"Invalid patch: expected 'Date:' or 'AuthorDate:' pseudo-header, "
|
|
|
|
f"found {header}.\nOnly 'medium' and 'fuller' formats are supported")
|
|
|
|
|
|
|
|
# skip possible extra headers before the message
|
[FIX] runbot_merge: my pager lied to me
Hopefully this is the last fix to the patcher. From the start of the
implementation I relied on the idea that `git show` was adding a line
composed of a single space (and a newline) before and after the patch
message, as that is what I observed in my terminal, and it's
consistent with RFC 3676 signatures (two dashes, a space, and a
newline).
Turns out that single space, while present in my terminal indeed, was
completely made up by `less(1)`. `git show` itself doesn't generate
that, neither does it appear when using most pagers, or even when
piping the output of `less` into something (a file, an other pager,
...). It's pretty much just something `less(1)` sends to a terminal
during interactive sessions to fuck with you.
Fixes #1037
2025-01-15 14:51:28 +07:00
|
|
|
while not next(lines, ' ').isspace():
|
2024-10-02 17:14:09 +07:00
|
|
|
continue
|
|
|
|
|
|
|
|
body = []
|
[FIX] runbot_merge: my pager lied to me
Hopefully this is the last fix to the patcher. From the start of the
implementation I relied on the idea that `git show` was adding a line
composed of a single space (and a newline) before and after the patch
message, as that is what I observed in my terminal, and it's
consistent with RFC 3676 signatures (two dashes, a space, and a
newline).
Turns out that single space, while present in my terminal indeed, was
completely made up by `less(1)`. `git show` itself doesn't generate
that, neither does it appear when using most pagers, or even when
piping the output of `less` into something (a file, an other pager,
...). It's pretty much just something `less(1)` sends to a terminal
during interactive sessions to fuck with you.
Fixes #1037
2025-01-15 14:51:28 +07:00
|
|
|
while (l := next(lines, '')) and l.startswith(' '):
|
2024-10-02 17:14:09 +07:00
|
|
|
body.append(l.removeprefix(' '))
|
|
|
|
|
|
|
|
# remainder should be the patch
|
|
|
|
patch = "".join(
|
|
|
|
line for line in lines
|
|
|
|
if not line.startswith("git --diff ")
|
|
|
|
if not line.startswith("index ")
|
|
|
|
)
|
|
|
|
return ParseResult(kind="show", author=author, committer=committer, message="".join(body).rstrip(), patch=patch)
|
|
|
|
|
|
|
|
|
|
|
|
def parse_format_patch(p: Patch) -> ParseResult:
|
|
|
|
m = message_from_string(p.patch, policy=policy.default)
|
|
|
|
if m.is_multipart():
|
|
|
|
raise ValidationError("multipart patches are not supported.")
|
|
|
|
|
|
|
|
name, email = parseaddr(m['from'])
|
|
|
|
author = (name, email, m['date'])
|
2024-11-18 18:37:44 +07:00
|
|
|
msg = re.sub(r'^\[PATCH( \d+/\d+)?\] ', '', m['subject'])
|
2024-11-19 16:15:25 +07:00
|
|
|
body, _, rest = m.get_payload().partition('---\n')
|
2024-10-02 17:14:09 +07:00
|
|
|
if body:
|
2024-11-28 19:46:44 +07:00
|
|
|
msg += '\n\n' + body.replace('\r\n', '\n')
|
2024-10-02 17:14:09 +07:00
|
|
|
|
|
|
|
# split off the signature, per RFC 3676 § 4.3.
|
|
|
|
# leave the diffstat in as it *should* not confuse tooling?
|
2024-11-28 19:46:44 +07:00
|
|
|
patch, _, _ = rest.partition("-- \n")
|
2024-10-02 17:14:09 +07:00
|
|
|
# git (diff, show, format-patch) adds command and index headers to every
|
|
|
|
# file header, which patch(1) chokes on, strip them... but maybe this should
|
|
|
|
# extract the udiff sections instead?
|
|
|
|
patch = re.sub(
|
|
|
|
"^(git --diff .*|index .*)\n",
|
|
|
|
"",
|
|
|
|
patch,
|
|
|
|
flags=re.MULTILINE,
|
|
|
|
)
|
|
|
|
return ParseResult(kind="format-patch", author=author, committer=author, message=msg, patch=patch)
|
|
|
|
|
|
|
|
|
|
|
|
class PatchFailure(Exception):
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
2024-11-28 21:41:42 +07:00
|
|
|
class PatchFile(models.TransientModel):
|
|
|
|
_name = "runbot_merge.patch.file"
|
|
|
|
_description = "metadata for single file to patch"
|
2025-02-24 14:01:48 +07:00
|
|
|
_order = "create_date desc"
|
2024-11-28 21:41:42 +07:00
|
|
|
|
|
|
|
name = fields.Char()
|
|
|
|
|
|
|
|
|
2024-10-02 17:14:09 +07:00
|
|
|
class Patch(models.Model):
|
|
|
|
_name = "runbot_merge.patch"
|
|
|
|
_inherit = ['mail.thread']
|
|
|
|
_description = "Unstaged direct-application patch"
|
|
|
|
|
|
|
|
active = fields.Boolean(default=True, tracking=True)
|
|
|
|
repository = fields.Many2one('runbot_merge.repository', required=True, tracking=True)
|
|
|
|
target = fields.Many2one('runbot_merge.branch', required=True, tracking=True)
|
|
|
|
commit = fields.Char(size=40, string="commit to cherry-pick, must be in-network", tracking=True)
|
|
|
|
|
|
|
|
patch = fields.Text(string="unified diff to apply", tracking=True)
|
|
|
|
format = fields.Selection([
|
|
|
|
("format-patch", "format-patch"),
|
|
|
|
("show", "show"),
|
|
|
|
], compute="_compute_patch_meta")
|
2024-11-28 21:41:42 +07:00
|
|
|
author = fields.Char(compute="_compute_patch_meta")
|
|
|
|
# TODO: should be a datetime, parse date
|
|
|
|
authordate = fields.Char(compute="_compute_patch_meta")
|
|
|
|
committer = fields.Char(compute="_compute_patch_meta")
|
|
|
|
# TODO: should be a datetime, parse date
|
|
|
|
commitdate = fields.Char(compute="_compute_patch_meta")
|
|
|
|
file_ids = fields.One2many(
|
|
|
|
"runbot_merge.patch.file",
|
|
|
|
compute="_compute_patch_meta",
|
|
|
|
)
|
2024-10-02 17:14:09 +07:00
|
|
|
message = fields.Text(compute="_compute_patch_meta")
|
|
|
|
|
|
|
|
_sql_constraints = [
|
|
|
|
('patch_contents_either', 'check ((commit is null) != (patch is null))', 'Either the commit or patch must be set, and not both.'),
|
|
|
|
]
|
|
|
|
|
|
|
|
@api.depends("patch")
|
|
|
|
def _compute_patch_meta(self) -> None:
|
2024-11-28 21:41:42 +07:00
|
|
|
File = self.env['runbot_merge.patch.file']
|
2024-10-02 17:14:09 +07:00
|
|
|
for p in self:
|
|
|
|
if r := p._parse_patch():
|
|
|
|
p.format = r.kind
|
2024-11-28 21:41:42 +07:00
|
|
|
match r.author:
|
|
|
|
case [name, email]:
|
|
|
|
p.author = f"{name} <{email}>"
|
|
|
|
case [name, email, date]:
|
|
|
|
p.author = f"{name} <{email}>"
|
|
|
|
p.authordate = date
|
|
|
|
match r.committer:
|
|
|
|
case [name, email]:
|
|
|
|
p.committer = f"{name} <{email}>"
|
|
|
|
case [name, email, date]:
|
|
|
|
p.committer = f"{name} <{email}>"
|
|
|
|
p.commitdate = date
|
|
|
|
p.file_ids = File.concat(*(
|
2025-01-29 19:29:53 +07:00
|
|
|
File.new({'name': m['file_to']})
|
2024-11-28 21:41:42 +07:00
|
|
|
for m in FILE_PATTERN.finditer(p.patch)
|
|
|
|
))
|
2024-10-02 17:14:09 +07:00
|
|
|
p.message = r.message
|
|
|
|
else:
|
2024-11-28 21:41:42 +07:00
|
|
|
p.update({
|
|
|
|
'format': False,
|
|
|
|
'author': False,
|
|
|
|
'authordate': False,
|
|
|
|
'committer': False,
|
|
|
|
'commitdate': False,
|
|
|
|
'file_ids': False,
|
|
|
|
'message': False,
|
|
|
|
})
|
2024-10-02 17:14:09 +07:00
|
|
|
|
|
|
|
def _parse_patch(self) -> ParseResult | None:
|
|
|
|
if not self.patch:
|
|
|
|
return None
|
|
|
|
|
|
|
|
if self.patch.startswith("commit "):
|
|
|
|
return parse_show(self)
|
|
|
|
elif self.patch.startswith("From "):
|
|
|
|
return parse_format_patch(self)
|
|
|
|
else:
|
|
|
|
raise ValidationError("Only `git show` and `git format-patch` formats are supported")
|
|
|
|
|
|
|
|
def _auto_init(self):
|
|
|
|
super()._auto_init()
|
|
|
|
self.env.cr.execute("""
|
|
|
|
CREATE INDEX IF NOT EXISTS runbot_merge_patch_active
|
|
|
|
ON runbot_merge_patch (target) WHERE active
|
|
|
|
""")
|
|
|
|
|
|
|
|
@api.model_create_multi
|
|
|
|
def create(self, vals_list):
|
|
|
|
if any(vals.get('active') is not False for vals in vals_list):
|
|
|
|
self.env.ref("runbot_merge.staging_cron")._trigger()
|
|
|
|
return super().create(vals_list)
|
|
|
|
|
|
|
|
def write(self, vals):
|
|
|
|
if vals.get("active") is not False:
|
|
|
|
self.env.ref("runbot_merge.staging_cron")._trigger()
|
|
|
|
return super().write(vals)
|
|
|
|
|
|
|
|
@api.constrains('patch')
|
|
|
|
def _validate_patch(self):
|
|
|
|
for p in self:
|
|
|
|
patch = p._parse_patch()
|
|
|
|
if not patch:
|
|
|
|
continue
|
|
|
|
|
|
|
|
has_files = False
|
|
|
|
for m in FILE_PATTERN.finditer(patch.patch):
|
|
|
|
has_files = True
|
2025-01-29 19:29:53 +07:00
|
|
|
if m['file_from'] != m['file_to'] and m['file_from'] != '/dev/null':
|
2024-10-02 17:14:09 +07:00
|
|
|
raise ValidationError("Only patches updating a file in place are supported, not creation, removal, or renaming.")
|
|
|
|
if not has_files:
|
|
|
|
raise ValidationError("Patches should have files they patch, found none.")
|
|
|
|
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
def _notify(self, subject: str | None, body: str, r: subprocess.CompletedProcess[str]) -> None:
|
|
|
|
self.message_post(
|
|
|
|
subject=subject,
|
|
|
|
body=Markup("\n").join(filter(None, [
|
|
|
|
Markup("<p>{}</p>").format(body),
|
|
|
|
r.stdout and Markup("<p>stdout:</p>\n<pre>\n{}</pre>").format(r.stdout),
|
|
|
|
r.stderr and Markup("<p>stderr:</p>\n<pre>\n{}</pre>").format(r.stderr),
|
|
|
|
])),
|
|
|
|
)
|
|
|
|
|
2024-10-02 17:14:09 +07:00
|
|
|
def _apply_patches(self, target: Branch) -> bool:
|
|
|
|
patches = self.search([('target', '=', target.id)], order='id asc')
|
[FIX] runbot_merge: handle missing patch commits
Commits can take some time to propagate through the network (I guess),
or human error can lead to the wrong commit being set.
Either way, because the entire thing was done using a single fetch in
`check=True` the cron job would fail entirely if any of the patch
commits was yet unavailable.
Update the updater to:
- fallback on individual fetches
- remove the patch from the set of applicable patch if we (still)
can't find its commit
I'd have hoped `fetch` could retrieve whatever it found, but
apparently the server just crashes out when it doesn't find the commit
we ask for, and `fetch` doesn't update anything.
No linked issue because I apparently forgot to jot it down (and only
remembered about this issue with the #1063 patching issue) but this
was reported by mat last week (2025-02-21) when they were wondering
why one of their patches was taking a while:
- At 0832 patch was created by automated script.
- At 0947, an attempt to apply was made, the commit was not found.
- At 1126, a second attempt was made but an other patch had been
created whose commit was not found, failing both.
- At 1255, there was a concurrency error ("cannot lock ref" on the
target branch).
- Finally at 1427 the patch was applied.
All in all it took 6 hours to apply the patch, which is 3-4 staging
cycles.
2025-02-25 20:00:45 +07:00
|
|
|
selected = len(patches)
|
|
|
|
if not selected:
|
2024-10-02 17:14:09 +07:00
|
|
|
return True
|
|
|
|
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
repo_info = {
|
|
|
|
r: {
|
|
|
|
'local': git.get_local(r).check(True).with_config(encoding="utf-8"),
|
|
|
|
'with_commit': self.browse(),
|
|
|
|
'target_head': None,
|
|
|
|
}
|
|
|
|
for r in patches.repository
|
|
|
|
}
|
|
|
|
for p in patches.filtered('commit'):
|
|
|
|
repo_info[p.repository]['with_commit'] |= p
|
|
|
|
|
|
|
|
for repo, info in repo_info.items():
|
|
|
|
r = info['local']
|
[FIX] runbot_merge: handle missing patch commits
Commits can take some time to propagate through the network (I guess),
or human error can lead to the wrong commit being set.
Either way, because the entire thing was done using a single fetch in
`check=True` the cron job would fail entirely if any of the patch
commits was yet unavailable.
Update the updater to:
- fallback on individual fetches
- remove the patch from the set of applicable patch if we (still)
can't find its commit
I'd have hoped `fetch` could retrieve whatever it found, but
apparently the server just crashes out when it doesn't find the commit
we ask for, and `fetch` doesn't update anything.
No linked issue because I apparently forgot to jot it down (and only
remembered about this issue with the #1063 patching issue) but this
was reported by mat last week (2025-02-21) when they were wondering
why one of their patches was taking a while:
- At 0832 patch was created by automated script.
- At 0947, an attempt to apply was made, the commit was not found.
- At 1126, a second attempt was made but an other patch had been
created whose commit was not found, failing both.
- At 1255, there was a concurrency error ("cannot lock ref" on the
target branch).
- Finally at 1427 the patch was applied.
All in all it took 6 hours to apply the patch, which is 3-4 staging
cycles.
2025-02-25 20:00:45 +07:00
|
|
|
remote = git.source_url(repo)
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
if r.check(False).fetch(
|
[FIX] runbot_merge: handle missing patch commits
Commits can take some time to propagate through the network (I guess),
or human error can lead to the wrong commit being set.
Either way, because the entire thing was done using a single fetch in
`check=True` the cron job would fail entirely if any of the patch
commits was yet unavailable.
Update the updater to:
- fallback on individual fetches
- remove the patch from the set of applicable patch if we (still)
can't find its commit
I'd have hoped `fetch` could retrieve whatever it found, but
apparently the server just crashes out when it doesn't find the commit
we ask for, and `fetch` doesn't update anything.
No linked issue because I apparently forgot to jot it down (and only
remembered about this issue with the #1063 patching issue) but this
was reported by mat last week (2025-02-21) when they were wondering
why one of their patches was taking a while:
- At 0832 patch was created by automated script.
- At 0947, an attempt to apply was made, the commit was not found.
- At 1126, a second attempt was made but an other patch had been
created whose commit was not found, failing both.
- At 1255, there was a concurrency error ("cannot lock ref" on the
target branch).
- Finally at 1427 the patch was applied.
All in all it took 6 hours to apply the patch, which is 3-4 staging
cycles.
2025-02-25 20:00:45 +07:00
|
|
|
remote,
|
|
|
|
f"+refs/heads/{target.name}:refs/heads/{target.name}",
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
*info['with_commit'].mapped('commit'),
|
2025-03-07 15:15:48 +07:00
|
|
|
no_tags=True,
|
[FIX] runbot_merge: handle missing patch commits
Commits can take some time to propagate through the network (I guess),
or human error can lead to the wrong commit being set.
Either way, because the entire thing was done using a single fetch in
`check=True` the cron job would fail entirely if any of the patch
commits was yet unavailable.
Update the updater to:
- fallback on individual fetches
- remove the patch from the set of applicable patch if we (still)
can't find its commit
I'd have hoped `fetch` could retrieve whatever it found, but
apparently the server just crashes out when it doesn't find the commit
we ask for, and `fetch` doesn't update anything.
No linked issue because I apparently forgot to jot it down (and only
remembered about this issue with the #1063 patching issue) but this
was reported by mat last week (2025-02-21) when they were wondering
why one of their patches was taking a while:
- At 0832 patch was created by automated script.
- At 0947, an attempt to apply was made, the commit was not found.
- At 1126, a second attempt was made but an other patch had been
created whose commit was not found, failing both.
- At 1255, there was a concurrency error ("cannot lock ref" on the
target branch).
- Finally at 1427 the patch was applied.
All in all it took 6 hours to apply the patch, which is 3-4 staging
cycles.
2025-02-25 20:00:45 +07:00
|
|
|
).returncode:
|
|
|
|
r.fetch(remote, f"+refs/heads/{target.name}:refs/heads/{target.name}", no_tags=True)
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
for p in info['with_commit']:
|
|
|
|
if (res := r.check(False).fetch(remote, p.commit, no_tags=True)).returncode:
|
[FIX] runbot_merge: handle missing patch commits
Commits can take some time to propagate through the network (I guess),
or human error can lead to the wrong commit being set.
Either way, because the entire thing was done using a single fetch in
`check=True` the cron job would fail entirely if any of the patch
commits was yet unavailable.
Update the updater to:
- fallback on individual fetches
- remove the patch from the set of applicable patch if we (still)
can't find its commit
I'd have hoped `fetch` could retrieve whatever it found, but
apparently the server just crashes out when it doesn't find the commit
we ask for, and `fetch` doesn't update anything.
No linked issue because I apparently forgot to jot it down (and only
remembered about this issue with the #1063 patching issue) but this
was reported by mat last week (2025-02-21) when they were wondering
why one of their patches was taking a while:
- At 0832 patch was created by automated script.
- At 0947, an attempt to apply was made, the commit was not found.
- At 1126, a second attempt was made but an other patch had been
created whose commit was not found, failing both.
- At 1255, there was a concurrency error ("cannot lock ref" on the
target branch).
- Finally at 1427 the patch was applied.
All in all it took 6 hours to apply the patch, which is 3-4 staging
cycles.
2025-02-25 20:00:45 +07:00
|
|
|
patches -= p
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
p._notify(None, f"Commit {p.commit} not found", res)
|
|
|
|
info['target_head'] = r.stdout().rev_list('-1', target.name).stdout.strip()
|
|
|
|
|
[FIX] runbot_merge: handle missing patch commits
Commits can take some time to propagate through the network (I guess),
or human error can lead to the wrong commit being set.
Either way, because the entire thing was done using a single fetch in
`check=True` the cron job would fail entirely if any of the patch
commits was yet unavailable.
Update the updater to:
- fallback on individual fetches
- remove the patch from the set of applicable patch if we (still)
can't find its commit
I'd have hoped `fetch` could retrieve whatever it found, but
apparently the server just crashes out when it doesn't find the commit
we ask for, and `fetch` doesn't update anything.
No linked issue because I apparently forgot to jot it down (and only
remembered about this issue with the #1063 patching issue) but this
was reported by mat last week (2025-02-21) when they were wondering
why one of their patches was taking a while:
- At 0832 patch was created by automated script.
- At 0947, an attempt to apply was made, the commit was not found.
- At 1126, a second attempt was made but an other patch had been
created whose commit was not found, failing both.
- At 1255, there was a concurrency error ("cannot lock ref" on the
target branch).
- Finally at 1427 the patch was applied.
All in all it took 6 hours to apply the patch, which is 3-4 staging
cycles.
2025-02-25 20:00:45 +07:00
|
|
|
# if some of the commits are not available (yet) schedule a new staging
|
|
|
|
# in case this is a low traffic period (so there might not be staging
|
|
|
|
# triggers every other minute
|
|
|
|
if len(patches) < selected:
|
|
|
|
self.env.ref('runbot_merge.staging_cron')._trigger(fields.Datetime.now() + timedelta(minutes=30))
|
2024-10-02 17:14:09 +07:00
|
|
|
|
|
|
|
for patch in patches:
|
|
|
|
patch.active = False
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
info = repo_info[patch.repository]
|
|
|
|
r = info['local']
|
|
|
|
sha = info['target_head']
|
2024-10-02 17:14:09 +07:00
|
|
|
|
|
|
|
_logger.info(
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
"Applying %s to %s:%r (%s@%s)",
|
2024-10-02 17:14:09 +07:00
|
|
|
patch,
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
patch.repository.name,
|
2024-10-02 17:14:09 +07:00
|
|
|
patch.target.display_name,
|
|
|
|
patch.repository.name,
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
sha,
|
2024-10-02 17:14:09 +07:00
|
|
|
)
|
2025-02-25 17:51:22 +07:00
|
|
|
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
# this tree should be available locally since we got `sha` from the
|
|
|
|
# local commit
|
|
|
|
t = r.get_tree(sha)
|
2024-10-02 17:14:09 +07:00
|
|
|
try:
|
2024-11-18 18:37:44 +07:00
|
|
|
if patch.commit:
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
c = patch._apply_commit(r, sha)
|
2024-11-18 18:37:44 +07:00
|
|
|
else:
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
c = patch._apply_patch(r, sha)
|
|
|
|
if t == r.get_tree(c):
|
|
|
|
raise PatchFailure(Markup(
|
2025-02-25 17:51:22 +07:00
|
|
|
"Patch results in an empty commit when applied, "
|
|
|
|
"it is likely a duplicate of a merged commit."
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
))
|
2024-10-02 17:14:09 +07:00
|
|
|
except Exception as e:
|
|
|
|
if isinstance(e, PatchFailure):
|
|
|
|
subject = "Unable to apply patch"
|
|
|
|
else:
|
|
|
|
subject = "Unknown error while trying to apply patch"
|
2024-11-18 18:37:44 +07:00
|
|
|
_logger.error("%s:\n%s", subject, str(e))
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
patch.message_post(
|
|
|
|
subject=subject,
|
|
|
|
# hack in order to get a formatted message from line 320 but
|
|
|
|
# a pre from git
|
|
|
|
# TODO: do better
|
|
|
|
body=e.args[0] if isinstance(e.args[0], Markup) else Markup("<pre>{}</pre>").format(e),
|
|
|
|
)
|
2024-10-02 17:14:09 +07:00
|
|
|
continue
|
|
|
|
|
|
|
|
# push patch by patch, avoids sync issues and in most cases we have 0~1 patches
|
|
|
|
res = r.check(False).stdout()\
|
|
|
|
.with_config(encoding="utf-8")\
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
.push(
|
|
|
|
git.source_url(patch.repository),
|
|
|
|
f"{c}:{target.name}",
|
|
|
|
f"--force-with-lease={target.name}:{sha}",
|
|
|
|
)
|
2024-10-02 17:14:09 +07:00
|
|
|
## one of the repos is out of consistency, loop around to new staging?
|
|
|
|
if res.returncode:
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
patch._notify(None, f"Unable to push result ({c})", res)
|
2024-10-02 17:14:09 +07:00
|
|
|
_logger.warning(
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
"Unable to push result of %s (%s)\nout:\n%s\nerr:\n%s",
|
|
|
|
patch, c, res.stdout, res.stderr,
|
2024-10-02 17:14:09 +07:00
|
|
|
)
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
else:
|
|
|
|
info['target_head'] = c
|
2024-10-02 17:14:09 +07:00
|
|
|
|
|
|
|
return True
|
|
|
|
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
def _apply_commit(self, r: git.Repo, parent: str) -> str:
|
2024-10-02 17:14:09 +07:00
|
|
|
r = r.check(True).stdout().with_config(encoding="utf-8")
|
|
|
|
target = r.show('--no-patch', '--pretty=%an%n%ae%n%ai%n%cn%n%ce%n%ci%n%B', self.commit)
|
|
|
|
# retrieve metadata of cherrypicked commit
|
|
|
|
author_name, author_email, author_date, committer_name, committer_email, committer_date, body =\
|
|
|
|
target.stdout.strip().split("\n", 6)
|
|
|
|
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
res = r.check(False).merge_tree(parent, self.commit)
|
2024-10-02 17:14:09 +07:00
|
|
|
if res.returncode:
|
|
|
|
_conflict_info, _, informational = res.stdout.partition('\n\n')
|
|
|
|
raise PatchFailure(informational)
|
|
|
|
|
|
|
|
return r.commit_tree(
|
|
|
|
tree=res.stdout.strip(),
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
parents=[parent],
|
2024-10-02 17:14:09 +07:00
|
|
|
message=body.strip(),
|
|
|
|
author=(author_name, author_email, author_date),
|
|
|
|
committer=(committer_name, committer_email, committer_date),
|
|
|
|
).stdout.strip()
|
|
|
|
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
def _apply_patch(self, r: git.Repo, parent: str) -> str:
|
2024-10-02 17:14:09 +07:00
|
|
|
p = self._parse_patch()
|
2024-11-18 18:37:44 +07:00
|
|
|
def reader(_r, f):
|
|
|
|
return pathlib.Path(tmpdir, f).read_text(encoding="utf-8")
|
|
|
|
|
|
|
|
prefix = 0
|
2025-01-29 19:29:53 +07:00
|
|
|
read = set()
|
|
|
|
patched = {}
|
2024-11-18 18:37:44 +07:00
|
|
|
for m in FILE_PATTERN.finditer(p.patch):
|
2025-01-29 19:29:53 +07:00
|
|
|
if not prefix and (m['prefix_a'] or m['file_from'] == '/dev/null') and m['prefix_b']:
|
2024-11-18 18:37:44 +07:00
|
|
|
prefix = 1
|
|
|
|
|
2025-01-29 19:29:53 +07:00
|
|
|
if m['file_from'] != '/dev/null':
|
|
|
|
read.add(m['file_from'])
|
|
|
|
patched[m['file_to']] = reader
|
2024-11-18 18:37:44 +07:00
|
|
|
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
archiver = r.stdout(True).with_config(encoding=None)
|
2024-10-02 17:14:09 +07:00
|
|
|
# if the parent is checked then we can't get rid of the kwarg and popen doesn't support it
|
|
|
|
archiver._config.pop('check', None)
|
|
|
|
archiver.runner = subprocess.Popen
|
2025-01-29 19:29:53 +07:00
|
|
|
with contextlib.ExitStack() as stack,\
|
2024-10-02 17:14:09 +07:00
|
|
|
tempfile.TemporaryDirectory() as tmpdir:
|
2025-01-29 19:29:53 +07:00
|
|
|
# if there's no file to *update*, `archive` will extract the entire
|
|
|
|
# tree which is unnecessary
|
|
|
|
if read:
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
out = stack.enter_context(archiver.archive(parent, *read))
|
2025-01-29 19:29:53 +07:00
|
|
|
tf = stack.enter_context(tarfile.open(fileobj=out.stdout, mode='r|'))
|
2025-03-14 20:21:47 +07:00
|
|
|
tf.extraction_filter = getattr(tarfile, 'data_filter', None)
|
2025-01-29 19:29:53 +07:00
|
|
|
tf.extractall(tmpdir)
|
2024-10-02 17:14:09 +07:00
|
|
|
patch = subprocess.run(
|
2024-11-19 16:15:25 +07:00
|
|
|
['patch', f'-p{prefix}', '--directory', tmpdir, '--verbose'],
|
2024-10-02 17:14:09 +07:00
|
|
|
input=p.patch,
|
|
|
|
stdout=subprocess.PIPE,
|
|
|
|
stderr=subprocess.PIPE,
|
|
|
|
encoding='utf-8',
|
|
|
|
)
|
|
|
|
if patch.returncode:
|
2024-11-19 16:15:25 +07:00
|
|
|
raise PatchFailure("\n---------\n".join(filter(None, [p.patch, patch.stdout.strip(), patch.stderr.strip()])))
|
2025-01-29 19:29:53 +07:00
|
|
|
new_tree = r.update_tree(self.target.name, patched)
|
2024-10-02 17:14:09 +07:00
|
|
|
|
|
|
|
return r.commit_tree(
|
|
|
|
tree=new_tree,
|
[FIX] runbot_merge: make patcher not rely on branches
This was the root cause of the incident of Feb 13/14: because the
patcher pushed to the local branch before pushing to the remote
failing to push to the remote would leave the local ref broken, as
`fetch("refs/heads/*:refs/heads/*")` apparently does not do non-ff
updates (which does make some sense I guess).
So in this case a staging finished, was pushed to the remote, then git
delayed the read side just enough that when the patcher looked up the
target it got the old commit. It applied a patch on top of that, tried
to push, and got a failure (non-ff update), which led the local and
remote branches divergent, and caused any further update of the local
reference branches to fail, thus every forward port to be blocked.
Using symbolic branches during patching was completely dumb (and
updating the local branch unnecessary), so switch the entire thing to
using just commits, and update a bunch of error reporting while at it.
2025-03-14 19:42:02 +07:00
|
|
|
parents=[parent],
|
2024-10-02 17:14:09 +07:00
|
|
|
message=p.message,
|
|
|
|
author=p.author,
|
|
|
|
committer=p.committer,
|
|
|
|
).stdout.strip()
|