ci: move nix and docker off the merge validation path (#5682)

* ci: move nix and docker off the merge validation path

Core ci.yml / Validate workspace / merge_group no longer schedule or require
Nix flake check or Docker image builds. Packaging lives on two standalone
workflows: nix.yml (flake check) and docker-image.yml (PR smoke build +
tag/release publish). Update scopes, topology tests, and agent docs.

* fix(ci): detect Docker publish from release inputs, not workflow_call event

Reusable workflow callers keep their own github.event_name (usually
workflow_dispatch), so matching event_name == workflow_call never fires and
stable GHCR publish would silently smoke-build only. Gate publish on tag push
or non-empty release_version / publish_latest inputs instead.

* fix(ci): keep smoke-only Docker sha tags off the publish path

Gate pr-/manual-sha metadata tags on publish != true so release-stable
workflow_dispatch callers do not push extra manual-sha-* tags to GHCR.
This commit is contained in:
PerishFire
2026-07-15 08:47:03 +00:00
committed by GitHub
parent 4c73aba565
commit 925db8d2af
9 changed files with 191 additions and 399 deletions
+2 -1
View File
@@ -24,8 +24,9 @@ GitHub automation uses two layers.
Business layer:
- Business workflows decide what happened and what should be requested next.
- `ci.yml` is the main low-privilege PR, merge-queue, and manual validation gate.
- `ci.yml` is the main low-privilege PR, merge-queue, and manual validation gate (application merge bar only).
- `ci.yml` should run validation, decide scopes, and produce typed handoff artifacts.
- Packaging checks are standalone and outside the merge gate: `nix.yml` (flake check) and `docker-image.yml` (image validate + publish). Do not re-attach them to `Validate workspace`.
- Business workflows should not perform trusted writes to PR comments or branches when a capability workflow can do it.
Atomic capability layer:
+2 -272
View File
@@ -56,14 +56,10 @@ jobs:
web_tests_required: ${{ steps.detect.outputs.web_tests_required }}
tools_dev_tests_required: ${{ steps.detect.outputs.tools_dev_tests_required }}
tools_pack_tests_required: ${{ steps.detect.outputs.tools_pack_tests_required }}
nix_validation_required: ${{ steps.detect.outputs.nix_validation_required }}
ui_p0_validation_required: ${{ steps.detect.outputs.ui_p0_validation_required }}
visual_validation_required: ${{ steps.detect.outputs.visual_validation_required }}
docker_validation_required: ${{ steps.detect.outputs.docker_validation_required }}
workspace_validation_required: ${{ steps.detect.outputs.workspace_validation_required }}
run_docker_build: ${{ steps.detect.outputs.run_docker_build }}
run_e2e_vitest: ${{ steps.detect.outputs.run_e2e_vitest }}
run_nix_validation: ${{ steps.detect.outputs.run_nix_validation }}
run_playwright_critical: ${{ steps.detect.outputs.run_playwright_critical }}
run_playwright_visual: ${{ steps.detect.outputs.run_playwright_visual }}
run_preflight: ${{ steps.detect.outputs.run_preflight }}
@@ -214,222 +210,6 @@ jobs:
echo "Changed-file blob guard passed at ${MAX_CHANGED_FILE_BYTES} bytes."
nix_validation:
name: Validate Nix flake
needs: [scopes, runners]
if: ${{ needs.scopes.outputs.run_nix_validation == 'true' }}
runs-on: ${{ fromJSON(needs.runners.outputs.runs_on).general_medium }}
timeout-minutes: 45
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@v6.0.2
- name: Install Nix
uses: cachix/install-nix-action@v27
with:
extra_nix_config: |
experimental-features = nix-command flakes
accept-flake-config = true
- name: Setup Node for Nix hash refresh
if: ${{ github.event_name == 'pull_request' }}
uses: actions/setup-node@v4
with:
node-version-file: package.json
- name: nix flake check
id: flake_check
continue-on-error: true
run: nix flake check --print-build-logs --keep-going
- name: Generate Nix hash refresh patch
id: hash_refresh
if: ${{ github.event_name == 'pull_request' && steps.flake_check.outcome == 'failure' }}
shell: bash
run: |
set -euo pipefail
out_dir="$RUNNER_TEMP/nix-hash-refresh"
mkdir -p "$out_dir"
status="update-failed"
if node --experimental-strip-types ./scripts/update-nix-pnpm-deps-hash.ts >"$out_dir/update.log" 2>&1; then
if git diff --quiet --exit-code -- nix/pnpm-deps.nix; then
status="no-change"
else
git diff -- nix/pnpm-deps.nix >"$out_dir/nix-pnpm-deps.patch"
cp nix/pnpm-deps.nix "$out_dir/pnpm-deps.nix"
status="patch-generated"
fi
fi
printf '{"status":"%s","runId":%s,"prNumber":%s,"headSha":"%s"}\n' \
"$status" \
'${{ github.run_id }}' \
'${{ github.event.pull_request.number }}' \
'${{ github.event.pull_request.head.sha }}' >"$out_dir/metadata.json"
echo "status=$status" >> "$GITHUB_OUTPUT"
- name: Create Nix hash refresh handoffs
id: nix_handoff
if: ${{ github.event_name == 'pull_request' && steps.flake_check.outcome == 'failure' && steps.hash_refresh.outputs.status != 'no-change' }}
env:
HASH_REFRESH_STATUS: ${{ steps.hash_refresh.outputs.status }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }}
REPOSITORY: ${{ github.repository }}
RUN_ID: ${{ github.run_id }}
shell: bash
run: |
set -euo pipefail
source_dir="$RUNNER_TEMP/nix-hash-refresh"
echo "comment_created=false" >> "$GITHUB_OUTPUT"
echo "autofix_created=false" >> "$GITHUB_OUTPUT"
if [ "$HASH_REFRESH_STATUS" = "patch-generated" ] && [ "$HEAD_REPOSITORY" = "$REPOSITORY" ]; then
handoff_id="nix-pnpm-deps"
handoff_root="$RUNNER_TEMP/handoff-autofix-$handoff_id"
handoff_dir="$(python3 .github/scripts/handoff.py dir autofix "$handoff_id" --root "$handoff_root")"
mkdir -p "$handoff_dir"
cp "$source_dir/nix-pnpm-deps.patch" "$handoff_dir/patch.diff"
jq -n \
--arg kind "autofix" \
--arg id "$handoff_id" \
--arg head "$HEAD_SHA" \
--arg base "$BASE_SHA" \
--arg message "chore(nix): refresh pnpm deps hash" \
--argjson schema_version 1 \
--argjson pr_number "$PR_NUMBER" \
--argjson run_id "$RUN_ID" \
--argjson allowed_paths '["nix/pnpm-deps.nix"]' \
'{schema_version: $schema_version, kind: $kind, id: $id, pr_number: $pr_number, head_sha: $head, base_sha: $base, run_id: $run_id, allowed_paths: $allowed_paths, commit_message: $message}' \
> "$handoff_dir/metadata.json"
python3 .github/scripts/handoff.py validate autofix "$handoff_dir" >/dev/null
{
echo "autofix_created=true"
echo "autofix_name=$(python3 .github/scripts/handoff.py artifact-name autofix "$handoff_id")"
echo "autofix_path=$handoff_root"
} >> "$GITHUB_OUTPUT"
fi
if [ "$HASH_REFRESH_STATUS" != "patch-generated" ] || [ "$HEAD_REPOSITORY" != "$REPOSITORY" ]; then
handoff_id="nix-hash-refresh"
handoff_root="$RUNNER_TEMP/handoff-comment-$handoff_id"
handoff_dir="$(python3 .github/scripts/handoff.py dir comment "$handoff_id" --root "$handoff_root")"
mkdir -p "$handoff_dir"
marker="<!-- nix-hash-refresh -->"
body_path="$handoff_dir/body.md"
{
printf '%s\n' "$marker"
if [ "$HASH_REFRESH_STATUS" = "patch-generated" ]; then
printf "A generated Nix hash refresh is available for this PR.\n\n"
printf "Apply the diff below with \`git apply\`:\n\n"
printf '```diff\n'
cat "$source_dir/nix-pnpm-deps.patch"
printf '\n```\n'
else
printf "The failed \`ci\` run attempted to refresh \`nix/pnpm-deps.nix\`, but it could not produce a hash-only patch.\n\n"
printf "Inspect the \`update.log\` file in the failed CI run artifacts for details.\n"
fi
} > "$body_path"
jq -n \
--arg kind "comment" \
--arg id "$handoff_id" \
--arg head "$HEAD_SHA" \
--arg base "$BASE_SHA" \
--arg marker "$marker" \
--argjson schema_version 1 \
--argjson pr_number "$PR_NUMBER" \
--argjson run_id "$RUN_ID" \
'{schema_version: $schema_version, kind: $kind, id: $id, pr_number: $pr_number, head_sha: $head, base_sha: $base, run_id: $run_id, marker: $marker}' \
> "$handoff_dir/metadata.json"
if [ -f "$source_dir/update.log" ]; then
cp "$source_dir/update.log" "$handoff_dir/update.log"
fi
python3 .github/scripts/handoff.py validate comment "$handoff_dir" >/dev/null
{
echo "comment_created=true"
echo "comment_name=$(python3 .github/scripts/handoff.py artifact-name comment "$handoff_id")"
echo "comment_path=$handoff_root"
} >> "$GITHUB_OUTPUT"
fi
- name: Upload Nix hash autofix handoff
if: ${{ steps.nix_handoff.outputs.autofix_created == 'true' }}
uses: actions/upload-artifact@v4
with:
name: ${{ steps.nix_handoff.outputs.autofix_name }}
path: ${{ steps.nix_handoff.outputs.autofix_path }}
if-no-files-found: error
retention-days: 14
- name: Upload Nix hash comment handoff
if: ${{ steps.nix_handoff.outputs.comment_created == 'true' }}
uses: actions/upload-artifact@v4
with:
name: ${{ steps.nix_handoff.outputs.comment_name }}
path: ${{ steps.nix_handoff.outputs.comment_path }}
if-no-files-found: error
retention-days: 14
- name: Summarize Nix hash refresh guidance
if: ${{ github.event_name == 'pull_request' && steps.flake_check.outcome == 'failure' }}
env:
HASH_REFRESH_STATUS: ${{ steps.hash_refresh.outputs.status }}
shell: bash
run: |
set -euo pipefail
case "${HASH_REFRESH_STATUS:-not-run}" in
patch-generated)
cat >> "$GITHUB_STEP_SUMMARY" <<'EOF'
## Generated Nix hash refresh
CI regenerated a hash-only patch for `nix/pnpm-deps.nix` and uploaded
it through the shared handoff contract.
- same-repo PRs: `autofix.atom.yml` will try to push the patch back to the
PR branch automatically and will publish patch guidance only if the
trusted writer cannot complete the autofix;
- fork PRs: `comment.atom.yml` will publish patch guidance on the PR.
EOF
;;
no-change)
cat >> "$GITHUB_STEP_SUMMARY" <<'EOF'
## Nix hash refresh unavailable
`nix flake check` failed, but `nix/pnpm-deps.nix` did not change after
running the hash refresh helper. Inspect the Nix build logs for a
non-hash failure.
EOF
;;
*)
cat >> "$GITHUB_STEP_SUMMARY" <<'EOF'
## Nix hash refresh failed
`nix flake check` failed and the helper could not generate a hash-only
patch. `comment.atom.yml` will publish the generated guidance when a
comment handoff is available.
EOF
;;
esac
- name: Fail when Nix validation fails
# Enforced everywhere EXCEPT pull_request. A stale `nix/pnpm-deps.nix` hash is the routine,
# self-healing consequence of any `pnpm-lock.yaml` change: the steps above already regenerate
# the hash and hand it to `autofix.atom.yml`, which pushes the refresh back to the PR branch.
# Failing the job on the PR turns a transient, auto-fixed hash drift into a red required-ish
# check that reads as a real CI failure downstream (BLOCKED mergeStateStatus / ci_status=failure
# to the review bot) — the same deadlock the `needs-validation` gate documents below. So the PR
# check stays green while the autofix lands, and the real gate runs at merge time: merge_group
# (and manual full runs) still fail closed, so a genuinely broken Nix build can never reach main.
if: ${{ steps.flake_check.outcome == 'failure' && github.event_name != 'pull_request' }}
run: exit 1
preflight:
name: Preflight
needs: [scopes, runners]
@@ -856,47 +636,6 @@ jobs:
if: ${{ steps.visual.outcome != 'success' }}
run: exit 1
docker_pr:
name: Docker image build
needs: [scopes, runners]
if: ${{ needs.scopes.outputs.run_docker_build == 'true' }}
runs-on: ${{ fromJSON(needs.runners.outputs.runs_on).general_medium }}
timeout-minutes: 30
permissions:
contents: read
packages: read
steps:
- name: Check out repository
uses: actions/checkout@v6.0.2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ghcr.io/${{ github.repository_owner }}/od
tags: |
type=sha,prefix=pr-${{ github.event.pull_request.number }}-sha-,format=short,enable=${{ github.event_name == 'pull_request' }}
type=sha,prefix=queue-sha-,format=short,enable=${{ github.event_name == 'merge_group' }}
type=sha,prefix=manual-sha-,format=short,enable=${{ github.event_name == 'workflow_dispatch' }}
- name: Build Docker image
uses: docker/build-push-action@v6
with:
context: .
file: deploy/Dockerfile
platforms: linux/amd64
push: false
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
NODE_IMAGE=public.ecr.aws/docker/library/node:24-alpine
RUNTIME_IMAGE=public.ecr.aws/docker/library/node:24-alpine
cache-from: type=gha
validate:
name: Validate workspace
needs:
@@ -904,7 +643,6 @@ jobs:
- runners
- static_gate
- preflight
- nix_validation
- workspace_unit_tests
- windows_tools_pack_payload_tests
- web_workspace_tests
@@ -913,7 +651,6 @@ jobs:
- ui_p0_smoke
- ui_p0
- playwright_visual
- docker_pr
if: ${{ always() }}
runs-on: ${{ fromJSON(needs.runners.outputs.runs_on).control }}
timeout-minutes: 5
@@ -933,17 +670,12 @@ jobs:
- name: Check workspace validation jobs
env:
NEEDS_JSON: ${{ toJSON(needs) }}
# `nix_validation` is advisory on pull_request and enforced only at merge time (see the
# `Fail when Nix validation fails` step). The gate honors that split below: on pull_request a
# red Nix job is ignored, on merge_group / manual full runs it is a hard requirement.
EVENT_NAME: ${{ github.event_name }}
run: |
set -euo pipefail
echo "$NEEDS_JSON" | jq .
failures="$(echo "$NEEDS_JSON" | jq -r --arg event "$EVENT_NAME" '
failures="$(echo "$NEEDS_JSON" | jq -r '
to_entries[]
| select(.value.result != "success" and .value.result != "skipped")
| select(.key != "nix_validation" or $event != "pull_request")
| "\(.key)=\(.value.result)"')"
if [ -n "$failures" ]; then
echo "Workspace validation failed:"
@@ -951,7 +683,7 @@ jobs:
exit 1
fi
required_misses="$(echo "$NEEDS_JSON" | jq -r --arg event "$EVENT_NAME" '
required_misses="$(echo "$NEEDS_JSON" | jq -r '
. as $needs |
($needs.scopes.outputs // {}) as $out |
def when($condition; $jobs): if $condition then $jobs else [] end;
@@ -965,8 +697,6 @@ jobs:
+ when($out.run_playwright_critical == "true"; ["playwright_critical"])
+ when($out.run_ui_p0 == "true"; ["ui_p0_smoke", "ui_p0"])
+ when($out.run_playwright_visual == "true"; ["playwright_visual"])
+ when($out.run_nix_validation == "true" and $event != "pull_request"; ["nix_validation"])
+ when($out.run_docker_build == "true"; ["docker_pr"])
)[]
| select(($needs[.].result // "missing") != "success")
| "\(.)=\($needs[.].result // "missing")"
+48 -8
View File
@@ -1,8 +1,9 @@
name: Docker image
# Phase 5 / spec §15.5 — multi-arch image builds.
# Standalone Docker packaging line (not part of core merge validation in ci.yml).
#
# Pushes to ghcr.io on release tags and when invoked by release automation.
# - pull_request / workflow_dispatch: smoke build only (push: false, amd64)
# - tag v*.*.* / workflow_call from release: multi-arch build + push to ghcr.io
on:
workflow_call:
@@ -24,6 +25,15 @@ on:
default: false
push:
tags: ['v*.*.*']
pull_request:
paths:
- "deploy/**"
- ".dockerignore"
- "package.json"
- "pnpm-lock.yaml"
- "pnpm-workspace.yaml"
- ".github/workflows/docker-image.yml"
workflow_dispatch:
jobs:
build-and-push:
@@ -38,7 +48,33 @@ jobs:
with:
ref: ${{ inputs.ref || github.sha }}
- name: Resolve publish mode
id: mode
shell: bash
env:
EVENT_NAME: ${{ github.event_name }}
RELEASE_VERSION: ${{ inputs.release_version }}
PUBLISH_LATEST: ${{ inputs.publish_latest }}
run: |
set -euo pipefail
# Publish on:
# - tag push (event_name=push with tags filter), or
# - reusable release call that passes release_version / publish_latest.
# Do NOT key on event_name == 'workflow_call': nested reusable workflows
# keep the *caller's* event name (usually workflow_dispatch), so that
# check never matches and stable GHCR publish would silently become a
# smoke-only build.
if [ "$EVENT_NAME" = "push" ] || [ -n "${RELEASE_VERSION:-}" ] || [ "${PUBLISH_LATEST:-}" = "true" ]; then
echo "publish=true" >> "$GITHUB_OUTPUT"
echo "platforms=linux/amd64,linux/arm64" >> "$GITHUB_OUTPUT"
else
# pull_request and standalone workflow_dispatch: smoke only
echo "publish=false" >> "$GITHUB_OUTPUT"
echo "platforms=linux/amd64" >> "$GITHUB_OUTPUT"
fi
- name: Set up QEMU
if: ${{ steps.mode.outputs.publish == 'true' }}
uses: docker/setup-qemu-action@v3
with:
platforms: linux/amd64,linux/arm64
@@ -47,7 +83,7 @@ jobs:
uses: docker/setup-buildx-action@v3
- name: Log in to GitHub Container Registry
if: github.event_name != 'pull_request'
if: ${{ steps.mode.outputs.publish == 'true' }}
uses: docker/login-action@v3
with:
registry: ghcr.io
@@ -62,11 +98,14 @@ jobs:
# spec §15.1 tag scheme:
# - vX.Y.Z push → :X.Y.Z + :latest
# - workflow_call (stable) → :<release_version> + optional :latest
# - PR / manual validate → local-only sha tags (only when not publishing)
tags: |
type=semver,pattern={{version}}
type=raw,value=${{ inputs.release_version }},enable=${{ inputs.release_version != '' }}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=raw,value=latest,enable=${{ inputs.publish_latest == true }}
type=sha,prefix=pr-${{ github.event.pull_request.number }}-sha-,format=short,enable=${{ steps.mode.outputs.publish != 'true' && github.event_name == 'pull_request' }}
type=sha,prefix=manual-sha-,format=short,enable=${{ steps.mode.outputs.publish != 'true' && github.event_name == 'workflow_dispatch' }}
labels: |
org.opencontainers.image.source=https://github.com/${{ github.repository }}
@@ -75,10 +114,8 @@ jobs:
with:
context: .
file: deploy/Dockerfile
# spec §15.1 — multi-arch single manifest
platforms: linux/amd64,linux/arm64
# Tags and release workflow calls publish.
push: ${{ github.event_name != 'pull_request' }}
platforms: ${{ steps.mode.outputs.platforms }}
push: ${{ steps.mode.outputs.publish == 'true' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
# The in-tree Dockerfile uses node:24-alpine + apk for build
@@ -86,11 +123,14 @@ jobs:
# from local builds. Spec §15.1 nominates bookworm-slim as
# the canonical base; switching is a follow-up that needs
# the Dockerfile's apk lines re-cast for apt.
build-args: |
NODE_IMAGE=public.ecr.aws/docker/library/node:24-alpine
RUNTIME_IMAGE=public.ecr.aws/docker/library/node:24-alpine
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Verify public GHCR pull access
if: github.event_name != 'pull_request'
if: ${{ steps.mode.outputs.publish == 'true' }}
shell: bash
env:
PUBLISHED_TAGS: ${{ steps.meta.outputs.tags }}
+55
View File
@@ -0,0 +1,55 @@
name: nix
# Standalone Nix packaging check. Not part of core merge validation (ci.yml).
# Triggered when flake/lock/nix inputs change; red here does not block merge.
on:
pull_request:
paths:
- "nix/**"
- "flake.nix"
- "flake.lock"
- "pnpm-lock.yaml"
- "pnpm-workspace.yaml"
- "package.json"
- "scripts/update-nix-pnpm-deps-hash.ts"
- ".github/workflows/nix.yml"
push:
branches: [main]
paths:
- "nix/**"
- "flake.nix"
- "flake.lock"
- "pnpm-lock.yaml"
- "pnpm-workspace.yaml"
- "package.json"
- "scripts/update-nix-pnpm-deps-hash.ts"
- ".github/workflows/nix.yml"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: nix-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
flake-check:
name: nix flake check
runs-on: ubuntu-24.04
timeout-minutes: 45
steps:
- name: Checkout
uses: actions/checkout@v6.0.2
- name: Install Nix
uses: cachix/install-nix-action@v27
with:
extra_nix_config: |
experimental-features = nix-command flakes
accept-flake-config = true
- name: nix flake check
run: nix flake check --print-build-logs --keep-going
+1 -1
View File
@@ -267,7 +267,7 @@ root `pnpm tools-pr` script without a new explicit maintainer decision.
- After package, workspace, or command-entry changes, run `pnpm install` so workspace links and generated dist entries stay fresh.
- For agent-stream / parser changes (`apps/daemon/src/claude-stream.ts`, `json-event-stream.ts`, `qoder-stream.ts`, etc.), replay a recorded session through the mock CLIs in `mocks/` to verify event shapes round-trip without burning provider budget. PATH-overlay activation: `export PATH="$PWD/mocks/bin:$PATH" OD_MOCKS_TRACE=<8-char-id> OD_MOCKS_NO_DELAY=1`. See `mocks/README.md` for the trace catalog and selection knobs.
- Treat every `pnpm-lock.yaml` change as requiring a Nix pnpm deps hash refresh check. `nix/pnpm-deps.nix` is a generated lock artifact; use `pnpm nix:update-hash` only when intentionally maintaining Nix packaging, then re-run `nix flake check --print-build-logs --keep-going`. Contributors without Nix can rely on the PR `Validate workspace` gate, which now uploads or auto-applies the generated hash-only fix when possible. A stale Nix hash is advisory on pull_request and never blocks the PR — the `nix_validation` job refreshes the hash via autofix while keeping the PR check green, and the hash is only enforced as a hard gate at merge time (`merge_group` and manual full runs fail closed).
- Treat every `pnpm-lock.yaml` change that affects Nix packaging as requiring a Nix pnpm deps hash refresh when you maintain the flake. `nix/pnpm-deps.nix` is a generated lock artifact; use `pnpm nix:update-hash` then re-run `nix flake check --print-build-logs --keep-going` locally. Standalone `.github/workflows/nix.yml` runs flake check when nix/lock inputs change; it is **not** part of core `ci.yml` / `Validate workspace` / merge queue. Docker image smoke/publish lives only in `.github/workflows/docker-image.yml` and is likewise outside the merge gate.
- Before marking regular work ready, run at least `pnpm guard` and `pnpm typecheck`, plus the package-scoped tests/builds that match the files changed. Do not use or add root `pnpm test`/`pnpm build` aliases.
- For local web runtime loops, prefer `pnpm tools-dev run web --daemon-port <port> --web-port <port>`.
- For e2e tests that need a tools-dev daemon/web runtime, use the shared tools-dev harness under `e2e/lib/tools-dev/` and the framework suite adapters (`e2e/lib/playwright/suite.ts`, `e2e/lib/vitest/suite.ts`). Do not hand-spawn `tools-dev` from test cases or duplicate lifecycle helpers under framework-specific folders.
+69 -45
View File
@@ -130,18 +130,24 @@ async function gitPatchId(mode: "--stable" | "--verbatim", diff: string): Promis
// Pull the two jq programs that the `validate` job's "Check workspace validation jobs" step runs
// (the blanket failure scan and the required-jobs scan) straight out of ci.yml, so the gate logic
// under test stays the real workflow source rather than a reimplementation. Both are invoked as
// `jq -r --arg event "$EVENT_NAME" '<program>'` and contain no single quotes, so the program is the
// text between the opening and the next `'`.
// `jq -r '<program>'` and contain no single quotes, so the program is the text between the opening
// and the next `'`.
function extractValidateGateJqPrograms(workflow: string): { failures: string; requiredMisses: string } {
const validate = sectionBetween(workflow, " validate:", " runtime_summary:");
const programs = [...validate.matchAll(/jq -r --arg event "\$EVENT_NAME" '([\s\S]*?)'/g)].map((match) => match[1] ?? "");
// Only the "Check workspace validation jobs" step — later validate steps also use jq.
const needsCheck = sectionBetween(
validate,
" - name: Check workspace validation jobs",
" - name: Block merge while the needs-validation label is present",
);
const programs = [...needsCheck.matchAll(/jq -r '([\s\S]*?)'/g)].map((match) => match[1] ?? "");
expect(programs).toHaveLength(2);
return { failures: programs[0] ?? "", requiredMisses: programs[1] ?? "" };
}
function runValidateGateJq(program: string, eventName: string, needs: unknown): Promise<string> {
function runValidateGateJq(program: string, needs: unknown): Promise<string> {
return new Promise((resolve, reject) => {
const child = execFile("jq", ["-r", "--arg", "event", eventName, program], { encoding: "utf8" }, (error, stdout, stderr) => {
const child = execFile("jq", ["-r", program], { encoding: "utf8" }, (error, stdout, stderr) => {
if (error) {
reject(Object.assign(error, { stdout, stderr }));
return;
@@ -153,11 +159,11 @@ function runValidateGateJq(program: string, eventName: string, needs: unknown):
}
// Mirrors the bash gate decision: the step exits non-zero when either jq program emits any line.
async function validateGatePasses(workflow: string, eventName: string, needs: unknown): Promise<boolean> {
async function validateGatePasses(workflow: string, needs: unknown): Promise<boolean> {
const { failures, requiredMisses } = extractValidateGateJqPrograms(workflow);
const [failureLines, missLines] = await Promise.all([
runValidateGateJq(failures, eventName, needs),
runValidateGateJq(requiredMisses, eventName, needs),
runValidateGateJq(failures, needs),
runValidateGateJq(requiredMisses, needs),
]);
return failureLines === "" && missLines === "";
}
@@ -315,7 +321,7 @@ describe("packaged smoke workflow", () => {
it("[P2] limits manual blob guard checks to changed files against main", async () => {
const workflow = await readFile(ciWorkflowPath, "utf8");
const blobGuard = sectionBetween(workflow, " static_gate:", " nix_validation:");
const blobGuard = sectionBetween(workflow, " static_gate:", " preflight:");
expect(blobGuard).toContain('${{ github.event_name }}" = "workflow_dispatch"');
expect(blobGuard).toContain("repos/${{ github.repository }}/compare/main...${{ github.sha }}");
@@ -332,7 +338,7 @@ describe("packaged smoke workflow", () => {
]);
const ciTrigger = sectionBetween(ciWorkflow, "on:", "\npermissions:");
const ciBlobGuard = sectionBetween(ciWorkflow, " static_gate:", " nix_validation:");
const ciBlobGuard = sectionBetween(ciWorkflow, " static_gate:", " preflight:");
const dockerTrigger = sectionBetween(dockerWorkflow, "on:", "\njobs:");
expect(ciTrigger).toContain("pull_request:");
@@ -342,8 +348,24 @@ describe("packaged smoke workflow", () => {
expect(ciBlobGuard).not.toContain('${{ github.event_name }}" = "push"');
expect(dockerTrigger).toContain("workflow_call:");
expect(dockerTrigger).toContain("tags: ['v*.*.*']");
expect(dockerTrigger).toContain("pull_request:");
// Publish stays tag/call only — no continuous main-branch image push.
expect(dockerTrigger).not.toContain("branches: [main]");
expect(dockerTrigger).not.toContain("- main");
expect(dockerTrigger).not.toMatch(/push:\s*\n\s*branches:/);
// Publish mode must not key only on event_name == workflow_call (caller keeps
// its own event name). Release calls pass release_version / publish_latest.
const dockerMode = sectionBetween(dockerWorkflow, "Resolve publish mode", "Set up QEMU");
expect(dockerMode).toContain("RELEASE_VERSION");
expect(dockerMode).toContain("PUBLISH_LATEST");
expect(dockerMode).toContain('[ "$EVENT_NAME" = "push" ]');
expect(dockerMode).toContain('[ -n "${RELEASE_VERSION:-}" ]');
// Shell condition must not treat literal workflow_call as the publish signal.
expect(dockerMode).not.toMatch(/\[\s*"\$EVENT_NAME"\s*=\s*"workflow_call"\s*\]/);
// Smoke-only sha tags must be disabled whenever publish mode is true (release
// callers are workflow_dispatch with release_version, and would otherwise push
// manual-sha-* alongside the real version tags).
expect(dockerWorkflow).toContain("steps.mode.outputs.publish != 'true' && github.event_name == 'pull_request'");
expect(dockerWorkflow).toContain("steps.mode.outputs.publish != 'true' && github.event_name == 'workflow_dispatch'");
expect(commentWorkflow).toContain("workflows: [ci]");
// comment.atom consumes merge_group runs too, so the needs-validation gate can surface a
// queue-ejection notice on the PR; autofix/report stay pull_request-only trusted consumers.
@@ -886,35 +908,45 @@ process.stdin.on("end", () => {
await expect(runScopesPrint("workflow_dispatch", { inputs: { ci_mode: "hot" } }, ["apps/web/src/app/page.tsx"])).resolves.toMatchObject({
ci_mode: "hot",
run_ui_p0: true,
run_nix_validation: false,
});
await expect(runScopesPrint("workflow_dispatch", { inputs: {} })).resolves.toMatchObject({
ci_mode: "full",
ui_p0_validation_required: true,
run_docker_build: true,
run_nix_validation: true,
run_ui_p0: true,
run_preflight: true,
});
await expect(runScopesPrint("merge_group", {})).resolves.toMatchObject({
ci_mode: "full",
ui_p0_validation_required: true,
run_docker_build: true,
run_nix_validation: true,
run_ui_p0: true,
run_preflight: true,
});
// Packaging (nix / docker) is no longer part of core scopes / Validate workspace.
for (const plan of await Promise.all([
runScopesPrint("merge_group", {}),
runScopesPrint("workflow_dispatch", { inputs: {} }),
runScopesPrint("pull_request", { pull_request: { number: 1 } }, ["apps/web/src/app/page.tsx"]),
])) {
expect(plan).not.toHaveProperty("run_nix_validation");
expect(plan).not.toHaveProperty("run_docker_build");
expect(plan).not.toHaveProperty("nix_validation_required");
expect(plan).not.toHaveProperty("docker_validation_required");
}
});
it("[P2] keeps the Validate workspace gate Nix-advisory on PRs and enforced at merge", async () => {
it("[P2] keeps packaging (nix/docker) off the core Validate workspace gate", async () => {
const workflow = await readFile(ciWorkflowPath, "utf8");
const validate = sectionBetween(workflow, " validate:", " runtime_summary:");
// The gate must read the event so it can treat a red nix_validation differently per surface.
expect(validate).toContain("EVENT_NAME: ${{ github.event_name }}");
expect(validate).toContain('select(.key != "nix_validation" or $event != "pull_request")');
expect(validate).toContain('when($out.run_nix_validation == "true" and $event != "pull_request"; ["nix_validation"])');
expect(workflow).not.toContain("nix_validation:");
expect(workflow).not.toContain("docker_pr:");
expect(validate).not.toContain("nix_validation");
expect(validate).not.toContain("docker_pr");
expect(validate).not.toContain("run_nix_validation");
expect(validate).not.toContain("run_docker_build");
expect(validate).toContain("Check workspace validation jobs");
const baseOutputs = {
run_nix_validation: "true",
run_preflight: "false",
run_workspace_unit_tests: "false",
run_windows_tools_pack_payload_tests: "false",
@@ -923,43 +955,34 @@ process.stdin.on("end", () => {
run_playwright_critical: "false",
run_ui_p0: "false",
run_playwright_visual: "false",
run_docker_build: "false",
};
const needsWithFailedNix = {
scopes: { result: "success", outputs: baseOutputs },
static_gate: { result: "success" },
nix_validation: { result: "failure" },
};
// Core gate only cares about app jobs — unknown packaging keys are ignored.
await expect(
validateGatePasses(workflow, {
scopes: { result: "success", outputs: baseOutputs },
static_gate: { result: "success" },
}),
).resolves.toBe(true);
// A stale/failed Nix hash is advisory on pull_request: the gate still passes while autofix heals it.
await expect(validateGatePasses(workflow, "pull_request", needsWithFailedNix)).resolves.toBe(true);
// ...but it is a hard gate at merge time and on manual full runs — fail closed, never reaching main.
await expect(validateGatePasses(workflow, "merge_group", needsWithFailedNix)).resolves.toBe(false);
await expect(validateGatePasses(workflow, "workflow_dispatch", needsWithFailedNix)).resolves.toBe(false);
// The PR exception is scoped to Nix only — any other failed required job still fails the gate on a PR.
const needsWithFailedWeb = {
scopes: { result: "success", outputs: { ...baseOutputs, run_web_workspace_tests: "true" } },
static_gate: { result: "success" },
nix_validation: { result: "success" },
web_workspace_tests: { result: "failure" },
};
await expect(validateGatePasses(workflow, "pull_request", needsWithFailedWeb)).resolves.toBe(false);
await expect(validateGatePasses(workflow, needsWithFailedWeb)).resolves.toBe(false);
});
it("[P2] routes default CI through cost-sensitive runner tiers", async () => {
const workflow = await readFile(ciWorkflowPath, "utf8");
const runners = sectionBetween(workflow, " runners:", " scopes:");
const scopes = sectionBetween(workflow, " scopes:", " static_gate:");
const staticGate = sectionBetween(workflow, " static_gate:", " nix_validation:");
const staticGate = sectionBetween(workflow, " static_gate:", " preflight:");
const workspaceUnitTests = sectionBetween(workflow, " workspace_unit_tests:", " windows_tools_pack_payload_tests:");
const webWorkspaceTests = sectionBetween(workflow, " web_workspace_tests:", " e2e_vitest:");
const e2eVitest = sectionBetween(workflow, " e2e_vitest:", " playwright_critical:");
const nixValidation = sectionBetween(workflow, " nix_validation:", " preflight:");
const preflight = sectionBetween(workflow, " preflight:", " workspace_unit_tests:");
const dockerPr = sectionBetween(workflow, " docker_pr:", " validate:");
const uiP0 = sectionBetween(workflow, " ui_p0:", " playwright_visual:");
const visual = sectionBetween(workflow, " playwright_visual:", " docker_pr:");
const visual = sectionBetween(workflow, " playwright_visual:", " validate:");
expect(runners).toContain("runs-on: ubuntu-24.04");
expect(runners).toContain("runs_on: ${{ steps.runners.outputs.runs_on }}");
@@ -977,10 +1000,8 @@ process.stdin.on("end", () => {
expect(e2eVitest).toContain("fromJSON(needs.runners.outputs.runs_on).js_hot");
expect(e2eVitest).toContain("toJSON(fromJSON(needs.runners.outputs.runs_on).js_hot)");
expect(e2eVitest).not.toContain('"od-persistent-ci"');
expect(nixValidation).toContain("fromJSON(needs.runners.outputs.runs_on).general_medium");
expect(preflight).toContain("fromJSON(needs.runners.outputs.runs_on).general_medium");
expect(preflight).toContain("toJSON(fromJSON(needs.runners.outputs.runs_on).general_medium)");
expect(dockerPr).toContain("fromJSON(needs.runners.outputs.runs_on).general_medium");
expect(uiP0).toContain("fromJSON(needs.runners.outputs.runs_on).ui_hot");
expect(uiP0).toContain("toJSON(fromJSON(needs.runners.outputs.runs_on).ui_hot)");
expect(uiP0).toContain("include: ${{ fromJSON(needs.scopes.outputs.ui_p0_matrix) }}");
@@ -1067,17 +1088,20 @@ process.stdin.on("end", () => {
readFile(handoffScriptPath, "utf8"),
]);
// Core ci still produces comment + report handoffs (needs-validation, visual).
// Packaging hash autofix left core ci with nix — no ci-produced autofix handoffs for now.
expect(ciWorkflow).toContain("handoff.py dir comment");
expect(ciWorkflow).toContain("handoff.py dir autofix");
expect(ciWorkflow).toContain("handoff.py dir report");
expect(ciWorkflow).toContain("handoff-comment-");
expect(ciWorkflow).toContain("handoff-autofix-");
expect(ciWorkflow).toContain("handoff-report-");
expect(ciWorkflow).not.toContain("handoff.py dir autofix");
expect(ciWorkflow).not.toContain("handoff-autofix-");
expect(ciWorkflow).not.toContain("nix-hash-autofix");
expect(ciWorkflow).not.toContain("visual-pr-comment");
expect(commentWorkflow).toContain("artifact-pattern comment");
expect(commentWorkflow).toContain("merge-multiple: false");
expect(commentWorkflow).toContain("pull-requests: write");
// Atom capability remains available for future producers; it is not required to be fed by ci.
expect(autofixWorkflow).toContain("artifact-pattern autofix");
expect(autofixWorkflow).toContain("allowed_paths");
expect(reportWorkflow).toContain("artifact-pattern report");
+13 -14
View File
@@ -241,19 +241,18 @@ installing the workspace.
## CI
`.github/workflows/ci.yml` owns Nix validation through the required
`Validate workspace` gate. Pull requests run `nix flake check` only when they
touch Nix inputs, daemon/web Nix build closures, or generated hash maintenance
workflows. Merge queue and manual full CI runs execute the full Nix path before
merge. The flake also filters each derivation down to only the workspace
packages it actually installs, so unrelated package/tool changes stay off the
slower Nix path and do not churn the other derivation's pnpm store hash.
Nix validation is a **standalone** workflow (`.github/workflows/nix.yml`), not
part of core merge validation (`ci.yml` / `Validate workspace` / merge queue).
It runs `nix flake check` on pull requests and `main` pushes that touch flake,
lock, or `nix/**` inputs (plus manual `workflow_dispatch`).
When a PR run fails because `nix/pnpm-deps.nix` is stale, the CI job also
tries to regenerate a hash-only patch:
Refresh a stale `nix/pnpm-deps.nix` locally:
- same-repo PRs get a bot-authored commit pushed back to the PR branch when
the generated patch only touches `nix/pnpm-deps.nix`;
- fork PRs get a PR comment plus a workflow artifact containing the patch;
- the failing run still stays red until the generated patch lands and a
fresh validation run passes.
```bash
pnpm nix:update-hash
nix flake check --print-build-logs --keep-going
```
The flake filters each derivation down to only the workspace packages it
actually installs, so unrelated package/tool changes stay off the slower Nix
path and do not churn the other derivation's pnpm store hash.
-1
View File
@@ -1276,7 +1276,6 @@ async function checkCiTopology(): Promise<boolean> {
"ci_mode: ${{ steps.detect.outputs.ci_mode }}",
"ui_p0_validation_required: ${{ steps.detect.outputs.ui_p0_validation_required }}",
"run_ui_p0: ${{ steps.detect.outputs.run_ui_p0 }}",
"run_nix_validation: ${{ steps.detect.outputs.run_nix_validation }}",
"ui_p0_matrix: ${{ steps.detect.outputs.ui_p0_matrix }}",
"visual_matrix: ${{ steps.detect.outputs.visual_matrix }}",
"include: ${{ fromJSON(needs.scopes.outputs.ui_p0_matrix) }}",
+1 -57
View File
@@ -10,18 +10,14 @@ type ScopeOutputs = {
web_tests_required: boolean;
tools_dev_tests_required: boolean;
tools_pack_tests_required: boolean;
nix_validation_required: boolean;
ui_p0_validation_required: boolean;
visual_validation_required: boolean;
docker_validation_required: boolean;
workspace_validation_required: boolean;
};
type ScopePlan = ScopeOutputs & {
ci_mode: CiMode;
run_docker_build: boolean;
run_e2e_vitest: boolean;
run_nix_validation: boolean;
run_playwright_critical: boolean;
run_playwright_visual: boolean;
run_preflight: boolean;
@@ -62,10 +58,8 @@ function createScopePlan(): ScopePlan {
web_tests_required: false,
tools_dev_tests_required: false,
tools_pack_tests_required: false,
nix_validation_required: false,
ui_p0_validation_required: false,
visual_validation_required: false,
docker_validation_required: false,
workspace_validation_required: false,
};
@@ -96,10 +90,8 @@ function createScopePlan(): ScopePlan {
outputs.web_tests_required = true;
outputs.tools_dev_tests_required = true;
outputs.tools_pack_tests_required = true;
outputs.nix_validation_required = true;
outputs.ui_p0_validation_required = true;
outputs.visual_validation_required = true;
outputs.docker_validation_required = true;
outputs.workspace_validation_required = true;
}
@@ -130,9 +122,7 @@ function createRunPlan(
return {
ci_mode: ciMode,
run_docker_build: isFull || outputs.docker_validation_required,
run_e2e_vitest: isFull || outputs.web_tests_required || outputs.ui_p0_validation_required,
run_nix_validation: isFull || outputs.nix_validation_required,
run_playwright_critical: isFull || (outputs.workspace_validation_required && !outputs.ui_p0_validation_required),
run_playwright_visual: isFull || outputs.visual_validation_required,
run_preflight: true,
@@ -249,14 +239,6 @@ function applyChangedFile(file: string, target: ScopeOutputs): void {
target.visual_validation_required = true;
}
if (isDockerRelevantFile(file)) {
target.docker_validation_required = true;
}
if (isNixRelevantFile(file)) {
target.nix_validation_required = true;
}
if (!isWorkspaceValidationExemptFile(file)) {
target.workspace_validation_required = true;
}
@@ -330,45 +312,6 @@ function isVisualRelevantFile(file: string): boolean {
);
}
function isDockerRelevantFile(file: string): boolean {
return (
startsWithAny(file, [
"deploy/",
]) ||
[
"package.json",
"pnpm-lock.yaml",
"pnpm-workspace.yaml",
".dockerignore",
".github/workflows/ci.yml",
".github/workflows/docker-image.yml",
].includes(file) ||
isWorkspaceManifestOrCiFile(file)
);
}
function isNixRelevantFile(file: string): boolean {
return (
startsWithAny(file, [
"nix/",
"packages/release/",
]) ||
[
"package.json",
"pnpm-lock.yaml",
"pnpm-workspace.yaml",
"flake.nix",
"flake.lock",
".github/scripts/handoff.py",
".github/workflows/autofix.atom.yml",
".github/workflows/ci.yml",
".github/workflows/comment.atom.yml",
"scripts/update-nix-pnpm-deps-hash.ts",
].includes(file) ||
isWorkspaceManifestOrCiFile(file)
);
}
function isWorkspaceValidationExemptFile(file: string): boolean {
return (
isDocumentationOrMetadataFile(file) ||
@@ -388,6 +331,7 @@ function isWorkspaceValidationExemptFile(file: string): boolean {
".github/workflows/comment.atom.yml",
".github/workflows/report.atom.yml",
".github/workflows/docker-image.yml",
".github/workflows/nix.yml",
].includes(file)
);
}