mirror of
https://github.com/nexu-io/open-design.git
synced 2026-10-27 09:51:12 +01:00
lefarcenandGitHub
25475defab
The `[P0] cold first Home run` packaged mac case has been red on almost
every beta build for eleven days (1 pass in 12 attempts) and nobody saw
it, because release-beta's smoke step is `continue-on-error: true`.
Four changes, in the order they matter:
1. Capture evidence before cleanup. A failing case uninstalled the app
and the next case deleted the runtime namespace, so the only surviving
trace was `logs: {skipped: true}`. The failure path now writes the
desktop log, a raw `/api/runs` diagnostics snapshot, and the error into
the release report before anything is torn down.
2. Give the wait a budget that matches the chain, split in two. 15s had to
cover cold project creation through rendered output; it is now 60s for
"the daemon owns a finished run" plus 30s for "the output surfaced", so
a timeout says which half broke instead of leaving three observations
that cannot falsify each other. Polling moved inside the page: every
sample used to cost a `tools-pack mac inspect` process, which ate about
a third of the old budget on its own.
3. Drop the injected 503. `POST /api/runs` does not answer
WORKSPACE_AUTHORITY_UNAVAILABLE — local run creation is deliberately
independent of Workspace membership availability — so the case spent
an extra create round trip plus a 500ms backoff on a branch production
cannot reach. Run-create retry semantics stay covered by
apps/web/tests/providers/sse.test.ts.
4. Make the exempted red audible. The exemption stays (at a 1/12 pass rate
a blocking gate would stall the daily beta), but `tools-release
write-report` now turns a failed smoke into a run annotation and a
caution banner at the top of the step summary.
76 lines
3.3 KiB
TypeScript
76 lines
3.3 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import { announceSwallowedSmoke, smokeFailed } from "../src/report/smoke-annotation.js";
|
|
|
|
const base = {
|
|
exempt: true,
|
|
outcome: "failure",
|
|
reportPath: "/tmp/release-report/mac_arm64",
|
|
suiteStatus: "failed",
|
|
target: "mac_arm64",
|
|
title: "mac_arm64 beta build",
|
|
version: "0.21.6-beta.7",
|
|
};
|
|
|
|
describe("swallowed smoke detection", () => {
|
|
it("treats a failed step outcome or a failed suite result as a failure", () => {
|
|
expect(smokeFailed("failure", "unknown")).toBe(true);
|
|
expect(smokeFailed("", "failed")).toBe(true);
|
|
expect(smokeFailed("success", "success")).toBe(false);
|
|
expect(smokeFailed("skipped", "unknown")).toBe(false);
|
|
expect(smokeFailed("", "")).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("swallowed smoke announcement", () => {
|
|
it("stays silent when the smoke passed or never ran", () => {
|
|
expect(announceSwallowedSmoke({ ...base, outcome: "success", suiteStatus: "success" }))
|
|
.toEqual({ annotation: null, banner: [], failed: false });
|
|
expect(announceSwallowedSmoke({ ...base, outcome: "skipped", suiteStatus: "unknown" }))
|
|
.toEqual({ annotation: null, banner: [], failed: false });
|
|
});
|
|
|
|
it("emits a run annotation and a leading banner when the failure was exempted", () => {
|
|
const announcement = announceSwallowedSmoke(base);
|
|
|
|
expect(announcement.failed).toBe(true);
|
|
expect(announcement.annotation).toContain("::error title=mac_arm64 packaged smoke failed::");
|
|
expect(announcement.annotation).toContain("continue-on-error kept the job green");
|
|
expect(announcement.banner[0]).toBe("> [!CAUTION]");
|
|
expect(announcement.banner.join("\n")).toContain("packaged smoke FAILED");
|
|
expect(announcement.banner.join("\n")).toContain("0.21.6-beta.7");
|
|
// The exemption is the whole reason this signal exists: a reader who sees a
|
|
// green job must be told, in the summary itself, that green is not a pass.
|
|
expect(announcement.banner.join("\n")).toContain("a green check on this job is not a pass");
|
|
expect(announcement.banner.join("\n")).toContain("/tmp/release-report/mac_arm64");
|
|
});
|
|
|
|
it("does not claim the job stayed green when the smoke was a blocking gate", () => {
|
|
const announcement = announceSwallowedSmoke({ ...base, exempt: false });
|
|
|
|
expect(announcement.failed).toBe(true);
|
|
expect(announcement.banner.join("\n")).toContain("blocking gate");
|
|
expect(announcement.banner.join("\n")).not.toContain("continue-on-error");
|
|
expect(announcement.annotation).not.toContain("continue-on-error");
|
|
});
|
|
|
|
it("escapes workflow-command delimiters so the annotation cannot be truncated", () => {
|
|
const announcement = announceSwallowedSmoke({
|
|
...base,
|
|
target: "mac:arm64,x64",
|
|
version: "1.0.0\nsecond line",
|
|
});
|
|
|
|
expect(announcement.annotation).toContain("title=mac%3Aarm64%2Cx64 packaged smoke failed::");
|
|
expect(announcement.annotation).toContain("1.0.0%0Asecond line");
|
|
expect(announcement.annotation).not.toContain("\n");
|
|
});
|
|
|
|
it("falls back to the target when no report title is configured", () => {
|
|
const announcement = announceSwallowedSmoke({ ...base, reportPath: "", title: " " });
|
|
|
|
expect(announcement.banner.join("\n")).toContain("mac_arm64 packaged smoke");
|
|
expect(announcement.banner.join("\n")).not.toContain("Evidence:");
|
|
});
|
|
});
|