mirror of
https://github.com/nexu-io/open-design.git
synced 2026-10-27 09:51:12 +01:00
ci: surface merge-queue CI failure ejections as a PR comment (#7229)
* ci: surface merge-queue CI failure ejections as a PR comment A failed workload in a merge_group run ejects the queued PR just as invisibly as a blocking label does: the required check fails on the queue's transient ref, the PR's own checks stay green, and the queue entry silently disappears. The needs-validation gate (#5519) announces its own ejections, but every other ejection left no trace on the PR. The validate job now produces a handoff/comment artifact after the gate has already failed in a merge group, naming the failed jobs and steps, a log excerpt of the failing assertions, the run, and the PRs batched ahead in the group. comment.atom.yml already consumes merge_group handoffs. merge_policy exposes whether it already announced a label ejection so a label-only ejection still gets exactly one comment. * ci: keep the merge-queue log excerpt working on gh >= 2.97 gh 2.97.0 (GHSA-3m3g-3wcr-px46) refuses to print an API body that contains terminal escape sequences unless --allow-escape-sequences is passed, and CI job logs are full of them, so the excerpt fetch returned nothing on a current gh. Older gh rejects the flag as unknown. Probe the flag once and pass it only when supported; the fake gh in the topology test now enforces both behaviors so dropping the probe goes red. * ci: make the merge-queue notice probe survive pipefail and replays Capturing gh api --help before grepping keeps an early grep exit from SIGPIPE-ing gh and reading as "flag unsupported" under pipefail; strip CRLF from job logs; never list the gate job itself as a failed job when a completed run is replayed. The fake gh pads its help text past the pipe buffer so the old probe shape stays red.
This commit is contained in:
lefarcen
@@ -670,6 +670,10 @@ jobs:
|
||||
if: ${{ github.event_name == 'merge_group' }}
|
||||
runs-on: ${{ fromJSON(needs.runners.outputs.runs_on).control }}
|
||||
timeout-minutes: 5
|
||||
# Tells the generic merge-queue failure notice in `validate` whether this job already
|
||||
# announced the ejection, so a label-only ejection gets exactly one comment.
|
||||
outputs:
|
||||
ejection_notice: ${{ steps.merge_blocking_label_gate.outputs.comment_created }}
|
||||
|
||||
steps:
|
||||
# The merge-blocking label gate produces a `handoff/comment` artifact when it ejects a
|
||||
@@ -922,6 +926,188 @@ jobs:
|
||||
path: ${{ runner.temp }}/hash-state.json
|
||||
key: ci-hash-${{ runner.os }}-${{ github.ref }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
|
||||
# A failed workload ejects the queued PR just as invisibly as a blocking label does: this
|
||||
# required check fails on the queue's transient ref, the PR's own checks stay green, and
|
||||
# its queue entry silently disappears. The merge-blocking label gate announces its own
|
||||
# ejections; this step announces every other one (failed or cancelled workloads, a
|
||||
# fail-closed policy error) so the author learns which jobs failed and where to look.
|
||||
# Best-effort by construction: it only runs after the gate has already failed and can
|
||||
# never soften, mask, or retry that result.
|
||||
- name: Checkout handoff helper
|
||||
if: ${{ failure() && github.event_name == 'merge_group' }}
|
||||
uses: actions/checkout@v6.0.2
|
||||
with:
|
||||
sparse-checkout: .github
|
||||
sparse-checkout-cone-mode: true
|
||||
|
||||
- name: Produce merge-queue failure notice handoff
|
||||
id: merge_queue_failure_notice
|
||||
if: ${{ failure() && github.event_name == 'merge_group' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPO: ${{ github.repository }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
NEEDS_JSON: ${{ toJSON(needs) }}
|
||||
# The queue names the PR that heads this group in its temporary ref
|
||||
# (`gh-readonly-queue/<base>/pr-<N>-<base sha>`); that PR owns this run's result.
|
||||
MERGE_GROUP_REF: ${{ github.event.merge_group.head_ref }}
|
||||
BASE_SHA: ${{ github.event.merge_group.base_sha }}
|
||||
HEAD_SHA: ${{ github.event.merge_group.head_sha }}
|
||||
# `true` when the merge-blocking label gate already produced its own ejection notice.
|
||||
LABEL_NOTICE_EMITTED: ${{ needs.merge_policy.outputs.ejection_notice }}
|
||||
# This job's own display name: it is still in progress while this step runs, but a
|
||||
# replay of a completed run would otherwise list the gate itself as a failed job.
|
||||
SELF_JOB_NAME: Validate workspace
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Which jobs the gate just rejected. Mirrors its two scans: any non-success,
|
||||
# non-skipped result plus any plan-required job that did not succeed. A broken plan
|
||||
# output must not break the notice, so the required-job scan is fault tolerant here.
|
||||
failed_jobs="$(jq -r '
|
||||
. as $needs |
|
||||
(try (($needs.plan.outputs.run // "{}") | fromjson) catch {}) as $run |
|
||||
(
|
||||
[ $needs | to_entries[] | select(.value.result != "success" and .value.result != "skipped") | .key ]
|
||||
+ [ (["plan"] + [$run | to_entries[] | select(.value) | .key])[]
|
||||
| select(($needs[.].result // "missing") != "success") ]
|
||||
) | unique[]' <<< "$NEEDS_JSON")"
|
||||
|
||||
pr="$(printf '%s\n' "$MERGE_GROUP_REF" | grep -oE 'pr-[0-9]+' | head -n 1 | grep -oE '[0-9]+' || true)"
|
||||
if [ -z "$pr" ]; then
|
||||
echo "::warning::merge_group ref '$MERGE_GROUP_REF' names no pull request; skipping the ejection notice."
|
||||
exit 0
|
||||
fi
|
||||
pr_json="$(gh api "repos/$REPO/pulls/$pr")"
|
||||
pr_head="$(jq -r '.head.sha' <<< "$pr_json")"
|
||||
pr_base="$(jq -r '.base.sha' <<< "$pr_json")"
|
||||
|
||||
# A label ejection is already announced by the gate's own notice (and a PR that still
|
||||
# carries a blocking label is about to get one). Drop the policy job from this notice
|
||||
# so a label-only ejection gets exactly one comment; a workload that failed alongside
|
||||
# the label is still reported.
|
||||
label_covered=false
|
||||
if [ "${LABEL_NOTICE_EMITTED:-}" = "true" ]; then
|
||||
label_covered=true
|
||||
elif jq -e '[.labels[]?.name] | any(. == "needs-validation" or . == "needs-maintainer-check")' <<< "$pr_json" >/dev/null; then
|
||||
label_covered=true
|
||||
fi
|
||||
skip_job_name=""
|
||||
if [ "$label_covered" = "true" ]; then
|
||||
skip_job_name="Merge policy"
|
||||
failed_jobs="$(printf '%s\n' "$failed_jobs" | grep -vx 'merge_policy' || true)"
|
||||
fi
|
||||
if [ -z "$failed_jobs" ]; then
|
||||
echo "Only the merge-blocking label gate failed; its own notice covers this ejection."
|
||||
exit 0
|
||||
fi
|
||||
echo "Failed validation jobs: $(printf '%s' "$failed_jobs" | tr '\n' ' ')"
|
||||
|
||||
# Job-level detail straight from the run: display names, the failing step, and a log
|
||||
# link per failed job. Read-only and best-effort; the notice still ships without it.
|
||||
jobs_json="$(gh api --paginate "repos/$REPO/actions/runs/$RUN_ID/jobs?per_page=100" | jq -s '[.[].jobs[]]' || echo '[]')"
|
||||
# Markdown code spans are intentionally literal in these single-quoted strings.
|
||||
# shellcheck disable=SC2016
|
||||
failed_lines="$(jq -r --arg skip "$skip_job_name" --arg self "$SELF_JOB_NAME" '
|
||||
.[]
|
||||
| select(.conclusion != null and .conclusion != "success" and .conclusion != "skipped" and .name != $skip and .name != $self)
|
||||
| "- **\(.name)** — \(.conclusion)"
|
||||
+ ([.steps[]? | select(.conclusion == "failure") | .name] | if length > 0 then " at `" + join("`, `") + "`" else "" end)
|
||||
+ " ([job log](\(.html_url)))"' <<< "$jobs_json")"
|
||||
if [ -z "$failed_lines" ]; then
|
||||
# shellcheck disable=SC2016
|
||||
failed_lines="$(printf '%s\n' "$failed_jobs" | sed -E 's/^/- `/; s/$/`/')"
|
||||
fi
|
||||
|
||||
# Pull the failing assertions out of the job logs (vitest `FAIL` lines, Playwright
|
||||
# `✘` lines, runner error annotations), capped so the comment stays readable.
|
||||
# CI logs are full of terminal escape sequences, and gh >= 2.97 refuses to print such
|
||||
# a body unless told otherwise (GHSA-3m3g-3wcr-px46) while older gh does not know the
|
||||
# flag — probe once so the excerpt works with either runner image.
|
||||
gh --version | head -n 1
|
||||
# Capture the help text first: `gh api --help | grep -q` would let grep exit early and
|
||||
# fail the pipeline under pipefail, silently disabling the flag.
|
||||
api_help="$(gh api --help 2>/dev/null || true)"
|
||||
escape_flag=""
|
||||
if grep -q -- '--allow-escape-sequences' <<< "$api_help"; then
|
||||
escape_flag="--allow-escape-sequences"
|
||||
fi
|
||||
esc="$(printf '\033')"
|
||||
excerpt=""
|
||||
for job_id in $(jq -r --arg skip "$skip_job_name" --arg self "$SELF_JOB_NAME" '.[] | select(.conclusion == "failure" and .name != $skip and .name != $self) | .id' <<< "$jobs_json" | head -n 4); do
|
||||
lines="$(gh api ${escape_flag:+"$escape_flag"} "repos/$REPO/actions/jobs/$job_id/logs" 2>/dev/null \
|
||||
| tr -d '\r' \
|
||||
| sed -E "s/${esc}\[[0-9;]*[A-Za-z]//g; s/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //" \
|
||||
| grep -E '^ ?FAIL |^[[:space:]]*[✘×] |^##\[error\]' \
|
||||
| grep -vE '^##\[error\]Process completed with exit code' \
|
||||
| sed -E 's/^##\[error\]//; s/^[[:space:]]+//' \
|
||||
| awk '!seen[$0]++' \
|
||||
| head -n 8 || true)"
|
||||
if [ -n "$lines" ]; then
|
||||
excerpt="${excerpt}${lines}"$'\n'
|
||||
fi
|
||||
done
|
||||
# Shards that fail on the same test would otherwise repeat the same lines per job.
|
||||
if [ -n "$excerpt" ]; then
|
||||
excerpt="$(printf '%s' "$excerpt" | awk '!seen[$0]++')"$'\n'
|
||||
fi
|
||||
|
||||
# Other PRs batched ahead of this one in the same group (their squashed commits sit
|
||||
# between the group base and head), so the author can tell whose change may have failed.
|
||||
group_note=""
|
||||
others="$(gh api "repos/$REPO/compare/$BASE_SHA...$HEAD_SHA" 2>/dev/null \
|
||||
| jq -r '.commits[].commit.message | split("\n")[0]' \
|
||||
| grep -oE '\(#[0-9]+\)$' | grep -oE '[0-9]+' | grep -vx "$pr" | sort -un | sed 's/^/#/' | paste -s -d ',' - | sed 's/,/, /g' || true)"
|
||||
if [ -n "$others" ]; then
|
||||
group_note=" and with the PRs queued ahead of it ($others)"
|
||||
fi
|
||||
|
||||
handoff_id="merge-queue-ci-failure-pr-$pr"
|
||||
marker="<!-- merge-queue-ci-failure -->"
|
||||
handoff_root="$RUNNER_TEMP/handoff-comment-$handoff_id"
|
||||
handoff_dir="$(python3 .github/scripts/handoff.py dir comment "$handoff_id" --root "$handoff_root")"
|
||||
mkdir -p "$handoff_dir"
|
||||
run_url="$GITHUB_SERVER_URL/$REPO/actions/runs/$RUN_ID"
|
||||
# Markdown code spans are intentionally literal in these single-quoted strings.
|
||||
# shellcheck disable=SC2016
|
||||
{
|
||||
printf '%s\n' "$marker"
|
||||
printf 'Ejected from the merge queue: CI failed on the queued merge of this PR.\n\n'
|
||||
printf 'The merge queue gate ([run %s](%s)) failed for the group headed by this PR. That run executes on the queue'"'"'s transient ref, so the failure never shows up in this PR'"'"'s own checks — they stay green, the queue entry simply disappears, and this notice is the only visible trace on the PR.\n\n' "$RUN_ID" "$run_url"
|
||||
printf 'Failed jobs:\n\n%s\n\n' "$failed_lines"
|
||||
if [ -n "$excerpt" ]; then
|
||||
printf 'Failure excerpt (best-effort, from the job logs):\n\n```text\n%s```\n\n' "$excerpt"
|
||||
fi
|
||||
printf 'The queued merge combines this PR with everything that landed on `main` after its last CI run%s. When this PR'"'"'s own checks are green, the failure is usually one of: a conflict with newer `main` (for example a test added or changed since this branch diverged), a PR ahead of it in the queue, or a flaky test.\n\n' "$group_note"
|
||||
printf 'To land this PR: open the run above and check whether the failure is related to this change. If it is, merge or rebase onto the latest `main`, fix, push, and add the PR back to the merge queue. If it is unrelated, add the PR back to the merge queue.\n'
|
||||
} > "$handoff_dir/body.md"
|
||||
jq -n \
|
||||
--arg kind "comment" \
|
||||
--arg id "$handoff_id" \
|
||||
--arg head "$pr_head" \
|
||||
--arg base "$pr_base" \
|
||||
--arg marker "$marker" \
|
||||
--argjson schema_version 1 \
|
||||
--argjson pr_number "$pr" \
|
||||
--argjson run_id "$RUN_ID" \
|
||||
'{schema_version: $schema_version, kind: $kind, id: $id, pr_number: $pr_number, head_sha: $head, base_sha: $base, run_id: $run_id, marker: $marker}' \
|
||||
> "$handoff_dir/metadata.json"
|
||||
python3 .github/scripts/handoff.py validate comment "$handoff_dir" >/dev/null
|
||||
{
|
||||
echo "comment_created=true"
|
||||
echo "comment_name=$(python3 .github/scripts/handoff.py artifact-name comment "$handoff_id")"
|
||||
echo "comment_path=$handoff_root"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "Produced merge-queue failure notice for PR #$pr."
|
||||
|
||||
- name: Upload merge-queue failure notice handoff
|
||||
if: ${{ failure() && steps.merge_queue_failure_notice.outputs.comment_created == 'true' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ steps.merge_queue_failure_notice.outputs.comment_name }}
|
||||
path: ${{ steps.merge_queue_failure_notice.outputs.comment_path }}
|
||||
|
||||
runtime_summary:
|
||||
name: Runtime summary
|
||||
needs:
|
||||
|
||||
Reference in New Issue
Block a user