mirror of
https://github.com/nexu-io/open-design.git
synced 2026-10-27 09:51:12 +01:00
Merge origin/main (post-7c8305f4) into reconcile branch
Brings in 10 new main commits: routine deep-link to specific conversations (#1508), Windows resource cache fix for Orbit templates, collapsible comment side panel (#1607), routines project radio polish, Copilot logo swap, and minor UI fixes. Conflicts resolved: - router.ts: garnet's home/view + marketplace routes + main's per-project conversationId deep-link field coexist on Route union - ProjectView.tsx: garnet's isPhantomDaemonRunMessage helper + main's isStoppableAssistantMessage helper both kept - ProjectView.run-cleanup.test.tsx: accepted HEAD (garnet's phantom-row regression test); main's three new tests for finalizeActiveAssistantMessagesOnStop / clearStreamingConversationMarker / shouldClearActiveRunRefs are queued as a follow-up TODO inline.
This commit is contained in:
lefarcen
@@ -0,0 +1,163 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
buildAssistantMessageUpdate,
|
||||
cancelTimedOutRun,
|
||||
createRunTimeoutError,
|
||||
dedupeDesignSystemIds,
|
||||
resolveDryRunDesignSystems,
|
||||
validateExplicitDesignSystemIds,
|
||||
} from "./batch-design-system-test.ts";
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
test("dedupeDesignSystemIds trims and preserves first-seen order", () => {
|
||||
assert.deepEqual(dedupeDesignSystemIds([" default ", "kami", "default", "", "kami "]), ["default", "kami"]);
|
||||
});
|
||||
|
||||
test("validateExplicitDesignSystemIds rejects unknown design system ids", () => {
|
||||
assert.throws(
|
||||
() => validateExplicitDesignSystemIds(["default", "typo"], ["default", "kami"]),
|
||||
/unknown design system id\(s\): typo/,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateExplicitDesignSystemIds returns the normalized explicit ids when all exist", () => {
|
||||
assert.deepEqual(validateExplicitDesignSystemIds([" default ", "kami", "default"], ["default", "kami"]), ["default", "kami"]);
|
||||
});
|
||||
|
||||
test("resolveDryRunDesignSystems normalizes explicit ids without daemon access", () => {
|
||||
assert.deepEqual(resolveDryRunDesignSystems({ designSystems: [" default ", "kami", "default"] }), ["default", "kami"]);
|
||||
});
|
||||
|
||||
test("resolveDryRunDesignSystems rejects --all-design-systems in dry-run mode", () => {
|
||||
assert.throws(
|
||||
() => resolveDryRunDesignSystems({ allDesignSystems: true }),
|
||||
/dry-run with --all-design-systems still requires daemon access/,
|
||||
);
|
||||
});
|
||||
|
||||
test("buildAssistantMessageUpdate persists the daemon run id on queued assistant rows", () => {
|
||||
assert.deepEqual(
|
||||
buildAssistantMessageUpdate({
|
||||
agentId: "claude",
|
||||
runId: "run-123",
|
||||
runStatus: "queued",
|
||||
createdAt: 123,
|
||||
}),
|
||||
{
|
||||
role: "assistant",
|
||||
content: "",
|
||||
agentId: "claude",
|
||||
agentName: "claude",
|
||||
runId: "run-123",
|
||||
runStatus: "queued",
|
||||
startedAt: 123,
|
||||
createdAt: 123,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("buildAssistantMessageUpdate keeps the daemon run id on final assistant updates", () => {
|
||||
assert.deepEqual(
|
||||
buildAssistantMessageUpdate({
|
||||
agentId: "claude",
|
||||
runId: "run-123",
|
||||
runStatus: "succeeded",
|
||||
createdAt: 123,
|
||||
content: "done",
|
||||
endedAt: 456,
|
||||
producedFiles: [{ path: "index.html" }],
|
||||
}),
|
||||
{
|
||||
role: "assistant",
|
||||
content: "done",
|
||||
agentId: "claude",
|
||||
agentName: "claude",
|
||||
runId: "run-123",
|
||||
runStatus: "succeeded",
|
||||
startedAt: 123,
|
||||
createdAt: 123,
|
||||
endedAt: 456,
|
||||
producedFiles: [{ path: "index.html" }],
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("cancelTimedOutRun posts to the daemon cancel endpoint for timeout errors", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const calls: Array<{ url: string; init: RequestInit | undefined }> = [];
|
||||
globalThis.fetch = (async (url: string | URL | Request, init?: RequestInit) => {
|
||||
calls.push({ url: String(url), init });
|
||||
return new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}) as typeof fetch;
|
||||
|
||||
try {
|
||||
const canceled = await cancelTimedOutRun("http://127.0.0.1:4321", createRunTimeoutError("run-123", 5000, "running"));
|
||||
assert.equal(canceled, true);
|
||||
assert.deepEqual(calls, [
|
||||
{
|
||||
url: "http://127.0.0.1:4321/api/runs/run-123/cancel",
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({}),
|
||||
},
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("cancelTimedOutRun ignores non-timeout errors", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let called = false;
|
||||
globalThis.fetch = (async () => {
|
||||
called = true;
|
||||
throw new Error("should not be called");
|
||||
}) as typeof fetch;
|
||||
|
||||
try {
|
||||
const canceled = await cancelTimedOutRun("http://127.0.0.1:4321", new Error("not a timeout"));
|
||||
assert.equal(canceled, false);
|
||||
assert.equal(called, false);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("CLI dry-run with explicit design systems succeeds without daemon discovery", () => {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--experimental-strip-types",
|
||||
path.join(HERE, "batch-design-system-test.ts"),
|
||||
"--prompt",
|
||||
"Test prompt",
|
||||
"--design-systems",
|
||||
"default,kami",
|
||||
"--dry-run",
|
||||
],
|
||||
{
|
||||
cwd: path.resolve(HERE, ".."),
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
OD_DAEMON_URL: "",
|
||||
OD_PORT: "",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(result.status, 0, result.stderr || result.stdout);
|
||||
assert.match(result.stdout, /design systems \(2\): default, kami/);
|
||||
assert.doesNotMatch(result.stdout + result.stderr, /cannot determine daemon URL/);
|
||||
});
|
||||
@@ -0,0 +1,775 @@
|
||||
#!/usr/bin/env node
|
||||
// Batch-create and optionally run one design brief across multiple design
|
||||
// systems through the public daemon API. Intended for fast visual regression
|
||||
// sweeps while iterating on DESIGN.md files.
|
||||
//
|
||||
// Usage (daemon must be running — e.g. `pnpm tools-dev`):
|
||||
// node --experimental-strip-types scripts/batch-design-system-test.ts \
|
||||
// --prompt "Design a pricing landing page for an AI notes app" \
|
||||
// --design-systems default,kami \
|
||||
// --skill open-design-landing \
|
||||
// --agent claude
|
||||
//
|
||||
// Config-file equivalent:
|
||||
// node --experimental-strip-types scripts/batch-design-system-test.ts --config batch-ds.json
|
||||
//
|
||||
// {
|
||||
// "prompt": "Design a pricing landing page for an AI notes app",
|
||||
// "designSystems": ["default", "kami"],
|
||||
// "skillId": "open-design-landing",
|
||||
// "agentId": "claude",
|
||||
// "metadata": { "kind": "prototype", "platform": "responsive" },
|
||||
// "concurrency": 2,
|
||||
// "timeoutMs": 900000,
|
||||
// "output": ".tmp/design-system-batch.jsonl"
|
||||
// }
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = path.resolve(HERE, '..');
|
||||
const RUN_PREFIX = 'ds-batch-';
|
||||
const FALLBACK_AGENT_ID = 'claude';
|
||||
const DEFAULT_KIND = 'prototype';
|
||||
const DEFAULT_PLATFORM = 'responsive';
|
||||
const DEFAULT_TIMEOUT_MS = 30 * 60 * 1000;
|
||||
const POLL_INTERVAL_MS = 1_500;
|
||||
|
||||
type ProjectKind = 'prototype' | 'deck' | 'template' | 'other' | 'image' | 'video' | 'audio';
|
||||
|
||||
interface ProjectMetadata {
|
||||
kind: ProjectKind;
|
||||
platform?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
interface BatchConfig {
|
||||
daemonUrl?: string | null;
|
||||
prompt?: string;
|
||||
promptFile?: string;
|
||||
designSystems?: string[];
|
||||
allDesignSystems?: boolean;
|
||||
agentId?: string;
|
||||
skillId?: string | null;
|
||||
model?: string | null;
|
||||
reasoning?: string | null;
|
||||
namePrefix?: string;
|
||||
metadata?: ProjectMetadata;
|
||||
customInstructions?: string | null;
|
||||
skipDiscoveryBrief?: boolean;
|
||||
startRuns?: boolean;
|
||||
wait?: boolean;
|
||||
concurrency?: number;
|
||||
timeoutMs?: number;
|
||||
output?: string | null;
|
||||
dryRun?: boolean;
|
||||
}
|
||||
|
||||
interface Args extends BatchConfig {
|
||||
configPath?: string;
|
||||
}
|
||||
|
||||
interface CreateProjectResponse {
|
||||
project: { id: string; name: string; designSystemId: string | null };
|
||||
conversationId?: string;
|
||||
}
|
||||
|
||||
interface RunCreateResponse {
|
||||
runId: string;
|
||||
}
|
||||
|
||||
interface RunStatusResponse {
|
||||
id: string;
|
||||
status: 'queued' | 'running' | 'succeeded' | 'failed' | 'canceled';
|
||||
exitCode?: number | null;
|
||||
error?: string | null;
|
||||
updatedAt?: number;
|
||||
}
|
||||
|
||||
interface ProjectFilesResponse {
|
||||
files?: unknown[];
|
||||
}
|
||||
|
||||
interface DesignSystemsResponse {
|
||||
designSystems?: Array<{ id: string; title?: string }>;
|
||||
systems?: Array<{ id: string; title?: string }>;
|
||||
}
|
||||
|
||||
interface AppConfigResponse {
|
||||
config?: {
|
||||
agentId?: string | null;
|
||||
};
|
||||
}
|
||||
|
||||
interface BatchResult {
|
||||
designSystemId: string;
|
||||
projectId: string;
|
||||
projectName: string;
|
||||
conversationId: string | null;
|
||||
runId: string | null;
|
||||
status: 'created' | RunStatusResponse['status'];
|
||||
daemonUrl: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface AssistantMessageUpdate {
|
||||
role: 'assistant';
|
||||
content: string;
|
||||
agentId: string;
|
||||
agentName: string;
|
||||
runId: string;
|
||||
runStatus: RunStatusResponse['status'];
|
||||
startedAt: number;
|
||||
createdAt: number;
|
||||
endedAt?: number;
|
||||
producedFiles?: unknown[];
|
||||
}
|
||||
|
||||
type RunTimeoutError = Error & {
|
||||
code: 'RUN_TIMEOUT';
|
||||
runId: string;
|
||||
lastStatus?: RunStatusResponse['status'];
|
||||
};
|
||||
|
||||
function buildRunPrompt(prompt: string, skipDiscoveryBrief: boolean): string {
|
||||
if (!skipDiscoveryBrief) return prompt;
|
||||
// The persisted skipDiscoveryBrief flag is understood by newer daemons, but
|
||||
// this script is often run against an already-started daemon while developing
|
||||
// the branch. Include the user-level escape hatch too so older daemons still
|
||||
// bypass the discovery form and actually build files.
|
||||
return `skip questions, just build. no questions, go.\n\n${prompt}`;
|
||||
}
|
||||
|
||||
function extractAgentTextFromSse(body: string): string {
|
||||
const chunks: string[] = [];
|
||||
for (const block of body.split(/\n\n+/)) {
|
||||
const eventLine = block.split('\n').find((line) => line.startsWith('event:'));
|
||||
if (eventLine?.slice('event:'.length).trim() !== 'agent') continue;
|
||||
const dataLines = block
|
||||
.split('\n')
|
||||
.filter((line) => line.startsWith('data:'))
|
||||
.map((line) => line.slice('data:'.length).trimStart());
|
||||
if (dataLines.length === 0) continue;
|
||||
try {
|
||||
const payload = JSON.parse(dataLines.join('\n')) as { type?: string; delta?: unknown };
|
||||
if (payload.type === 'text_delta' && typeof payload.delta === 'string') chunks.push(payload.delta);
|
||||
} catch {
|
||||
// Ignore malformed/non-JSON event payloads.
|
||||
}
|
||||
}
|
||||
return chunks.join('');
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.log(`Usage: node --experimental-strip-types scripts/batch-design-system-test.ts [opts]
|
||||
|
||||
Creates one project per design system with the same prompt. By default it also
|
||||
starts an agent run and waits for every run to finish.
|
||||
|
||||
Required input:
|
||||
--prompt <text> Brief to run for every design system.
|
||||
--prompt-file <path> Read the brief from a file instead of --prompt.
|
||||
--design-systems <ids> Comma-separated design system ids, e.g. default,kami.
|
||||
--design-system <id> Add one design system id. Can be repeated.
|
||||
--all-design-systems Use every design system reported by the daemon.
|
||||
--config <path> JSON config file. CLI flags override config values.
|
||||
|
||||
Run/project options:
|
||||
--daemon <url> Daemon base URL. Falls back to OD_DAEMON_URL,
|
||||
OD_PORT, then tools-dev status discovery.
|
||||
--agent <id> Agent id for /api/runs. Default: daemon's saved
|
||||
app config agentId, then ${FALLBACK_AGENT_ID}.
|
||||
--skill <id|null> Skill/design-template id to bind to each project.
|
||||
--model <id> Optional per-run model override.
|
||||
--reasoning <id> Optional per-run reasoning override.
|
||||
--kind <kind> Project kind metadata. Default: ${DEFAULT_KIND}.
|
||||
--platform <platform> Project platform metadata. Default: ${DEFAULT_PLATFORM}.
|
||||
--metadata <json> Extra metadata JSON merged into project metadata.
|
||||
--custom-instructions <txt> Project-level custom instructions.
|
||||
--name-prefix <text> Project name prefix. Default includes timestamp.
|
||||
--no-skip-discovery Do not set skipDiscoveryBrief on created projects.
|
||||
|
||||
Batch options:
|
||||
--concurrency <n> Parallel project/run count. Default: 1.
|
||||
--timeout-ms <n> Per-run wait timeout. Default: ${DEFAULT_TIMEOUT_MS}.
|
||||
--create-only Create projects but do not start agent runs.
|
||||
--no-wait Start runs and return after run ids are created.
|
||||
--output <path> Write JSONL results to a file.
|
||||
--dry-run Print the resolved plan without calling daemon.
|
||||
-h, --help Show this help.
|
||||
|
||||
Example config:
|
||||
{
|
||||
"prompt": "Build a responsive pricing page for an AI notes app",
|
||||
"designSystems": ["default", "kami"],
|
||||
"skillId": "open-design-landing",
|
||||
"agentId": "claude",
|
||||
"metadata": { "kind": "prototype", "platform": "responsive" },
|
||||
"concurrency": 2,
|
||||
"timeoutMs": 900000,
|
||||
"output": ".tmp/design-system-batch.jsonl"
|
||||
}
|
||||
`);
|
||||
}
|
||||
|
||||
function parseCsv(value: string): string[] {
|
||||
return value
|
||||
.split(',')
|
||||
.map((part) => part.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
export function dedupeDesignSystemIds(ids: string[]): string[] {
|
||||
return [...new Set(ids.map((id) => id.trim()).filter(Boolean))];
|
||||
}
|
||||
|
||||
export function validateExplicitDesignSystemIds(requestedIds: string[], availableIds: string[]): string[] {
|
||||
const requested = dedupeDesignSystemIds(requestedIds);
|
||||
const available = new Set(availableIds.map((id) => id.trim()).filter(Boolean));
|
||||
const unknown = requested.filter((id) => !available.has(id));
|
||||
if (unknown.length > 0) {
|
||||
throw new Error(`unknown design system id(s): ${unknown.join(', ')}`);
|
||||
}
|
||||
return requested;
|
||||
}
|
||||
|
||||
export function buildAssistantMessageUpdate(params: {
|
||||
agentId: string;
|
||||
runId: string;
|
||||
runStatus: RunStatusResponse['status'];
|
||||
createdAt: number;
|
||||
content?: string;
|
||||
endedAt?: number;
|
||||
producedFiles?: unknown[];
|
||||
}): AssistantMessageUpdate {
|
||||
const { agentId, runId, runStatus, createdAt, content = '', endedAt, producedFiles } = params;
|
||||
return {
|
||||
role: 'assistant',
|
||||
content,
|
||||
agentId,
|
||||
agentName: agentId,
|
||||
runId,
|
||||
runStatus,
|
||||
startedAt: createdAt,
|
||||
createdAt,
|
||||
...(endedAt !== undefined ? { endedAt } : {}),
|
||||
...(producedFiles !== undefined ? { producedFiles } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function createRunTimeoutError(
|
||||
runId: string,
|
||||
timeoutMs: number,
|
||||
lastStatus?: RunStatusResponse['status'],
|
||||
): RunTimeoutError {
|
||||
const error = new Error(
|
||||
`run timed out after ${timeoutMs}ms${lastStatus ? ` (last status: ${lastStatus})` : ''}`,
|
||||
) as RunTimeoutError;
|
||||
error.code = 'RUN_TIMEOUT';
|
||||
error.runId = runId;
|
||||
if (lastStatus !== undefined) error.lastStatus = lastStatus;
|
||||
return error;
|
||||
}
|
||||
|
||||
function isRunTimeoutError(error: unknown): error is RunTimeoutError {
|
||||
return (
|
||||
!!error &&
|
||||
typeof error === 'object' &&
|
||||
(error as { code?: unknown }).code === 'RUN_TIMEOUT' &&
|
||||
typeof (error as { runId?: unknown }).runId === 'string'
|
||||
);
|
||||
}
|
||||
|
||||
function parseJsonObject(value: string, label: string): Record<string, unknown> {
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
throw new Error(`${label} must be a JSON object`);
|
||||
}
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): Args {
|
||||
const out: Args = {};
|
||||
const designSystems: string[] = [];
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const flag = argv[i];
|
||||
const next = () => {
|
||||
const value = argv[++i];
|
||||
if (!value) throw new Error(`${flag} requires a value`);
|
||||
return value;
|
||||
};
|
||||
if (flag === '--config') out.configPath = next();
|
||||
else if (flag === '--daemon' || flag === '--daemon-url') out.daemonUrl = next();
|
||||
else if (flag === '--prompt') out.prompt = next();
|
||||
else if (flag === '--prompt-file') out.promptFile = next();
|
||||
else if (flag === '--design-systems') designSystems.push(...parseCsv(next()));
|
||||
else if (flag === '--design-system') designSystems.push(next());
|
||||
else if (flag === '--all-design-systems') out.allDesignSystems = true;
|
||||
else if (flag === '--agent') out.agentId = next();
|
||||
else if (flag === '--skill') {
|
||||
const value = next();
|
||||
out.skillId = value === 'null' || value === 'none' ? null : value;
|
||||
} else if (flag === '--model') out.model = next();
|
||||
else if (flag === '--reasoning') out.reasoning = next();
|
||||
else if (flag === '--kind') {
|
||||
out.metadata = { ...(out.metadata ?? { kind: DEFAULT_KIND }), kind: next() as ProjectKind };
|
||||
} else if (flag === '--platform') {
|
||||
out.metadata = { ...(out.metadata ?? { kind: DEFAULT_KIND }), platform: next() };
|
||||
} else if (flag === '--metadata') {
|
||||
const metadata = parseJsonObject(next(), '--metadata');
|
||||
out.metadata = { ...(out.metadata ?? { kind: DEFAULT_KIND }), ...metadata } as ProjectMetadata;
|
||||
} else if (flag === '--custom-instructions') out.customInstructions = next();
|
||||
else if (flag === '--name-prefix') out.namePrefix = next();
|
||||
else if (flag === '--no-skip-discovery') out.skipDiscoveryBrief = false;
|
||||
else if (flag === '--concurrency') out.concurrency = positiveInteger(next(), '--concurrency');
|
||||
else if (flag === '--timeout-ms') out.timeoutMs = positiveInteger(next(), '--timeout-ms');
|
||||
else if (flag === '--create-only') out.startRuns = false;
|
||||
else if (flag === '--no-wait') out.wait = false;
|
||||
else if (flag === '--output') out.output = next();
|
||||
else if (flag === '--dry-run') out.dryRun = true;
|
||||
else if (flag === '-h' || flag === '--help') {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
} else {
|
||||
throw new Error(`unknown flag: ${flag}`);
|
||||
}
|
||||
}
|
||||
if (designSystems.length > 0) out.designSystems = designSystems;
|
||||
return out;
|
||||
}
|
||||
|
||||
function positiveInteger(value: string, label: string): number {
|
||||
const n = Number(value);
|
||||
if (!Number.isInteger(n) || n <= 0) throw new Error(`${label} must be a positive integer`);
|
||||
return n;
|
||||
}
|
||||
|
||||
async function readConfig(pathLike: string | undefined): Promise<BatchConfig> {
|
||||
if (!pathLike) return {};
|
||||
const absolute = path.resolve(REPO_ROOT, pathLike);
|
||||
const parsed = JSON.parse(await readFile(absolute, 'utf8')) as unknown;
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
throw new Error(`config must be a JSON object: ${pathLike}`);
|
||||
}
|
||||
return parsed as BatchConfig;
|
||||
}
|
||||
|
||||
function mergeConfig(fileConfig: BatchConfig, cli: Args): BatchConfig {
|
||||
const merged: BatchConfig = { ...fileConfig, ...cli };
|
||||
delete (merged as { configPath?: string }).configPath;
|
||||
merged.metadata = {
|
||||
kind: DEFAULT_KIND,
|
||||
platform: DEFAULT_PLATFORM,
|
||||
...(fileConfig.metadata ?? {}),
|
||||
...(cli.metadata ?? {}),
|
||||
};
|
||||
if (cli.designSystems) merged.designSystems = cli.designSystems;
|
||||
return merged;
|
||||
}
|
||||
|
||||
function isDiscoverablePort(value: string | undefined): value is string {
|
||||
if (value == null || value.length === 0) return false;
|
||||
const n = Number(value);
|
||||
return Number.isInteger(n) && n > 0 && n < 65536;
|
||||
}
|
||||
|
||||
async function discoverDaemonUrlFromToolsDev(): Promise<string | null> {
|
||||
return await new Promise<string | null>((resolve) => {
|
||||
let child;
|
||||
try {
|
||||
child = spawn('pnpm', ['--silent', 'exec', 'tools-dev', 'status', '--json'], {
|
||||
cwd: REPO_ROOT,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
} catch {
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
let stdout = '';
|
||||
child.stdout?.on('data', (chunk: Buffer | string) => {
|
||||
stdout += typeof chunk === 'string' ? chunk : chunk.toString('utf8');
|
||||
});
|
||||
child.stderr?.resume();
|
||||
child.on('error', () => resolve(null));
|
||||
child.on('exit', () => resolve(extractDaemonUrlFromStatusOutput(stdout)));
|
||||
});
|
||||
}
|
||||
|
||||
function extractDaemonUrlFromStatusOutput(stdout: string): string | null {
|
||||
for (let i = stdout.indexOf('{'); i !== -1; i = stdout.indexOf('{', i + 1)) {
|
||||
try {
|
||||
const parsed = JSON.parse(stdout.slice(i)) as {
|
||||
apps?: { daemon?: { url?: string | null } };
|
||||
url?: string | null;
|
||||
};
|
||||
const url = parsed.apps?.daemon?.url ?? parsed.url ?? null;
|
||||
if (typeof url === 'string' && url.length > 0) return url;
|
||||
} catch {
|
||||
// try the next possible JSON object
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function resolveDaemonUrl(config: BatchConfig): Promise<string> {
|
||||
if (config.daemonUrl) return config.daemonUrl;
|
||||
if (process.env.OD_DAEMON_URL) return process.env.OD_DAEMON_URL;
|
||||
if (isDiscoverablePort(process.env.OD_PORT)) return `http://127.0.0.1:${process.env.OD_PORT}`;
|
||||
const discovered = await discoverDaemonUrlFromToolsDev();
|
||||
if (discovered) return discovered;
|
||||
throw new Error('cannot determine daemon URL; start `pnpm tools-dev` or pass --daemon <url>');
|
||||
}
|
||||
|
||||
async function api<T = unknown>(daemonUrl: string, method: string, pathPart: string, body?: unknown): Promise<T> {
|
||||
const url = `${daemonUrl.replace(/\/$/, '')}${pathPart}`;
|
||||
const init: RequestInit = { method };
|
||||
if (body !== undefined) {
|
||||
init.headers = { 'content-type': 'application/json' };
|
||||
init.body = JSON.stringify(body);
|
||||
}
|
||||
let resp: Response;
|
||||
try {
|
||||
resp = await fetch(url, init);
|
||||
} catch (err) {
|
||||
throw new Error(`cannot reach daemon at ${daemonUrl}: ${(err as Error).message || String(err)}`);
|
||||
}
|
||||
if (!resp.ok) {
|
||||
const text = await resp.text().catch(() => '');
|
||||
throw new Error(`${method} ${pathPart} → ${resp.status}: ${text}`);
|
||||
}
|
||||
if (resp.status === 204) return undefined as T;
|
||||
return (await resp.json()) as T;
|
||||
}
|
||||
|
||||
async function resolvePrompt(config: BatchConfig): Promise<string> {
|
||||
if (typeof config.prompt === 'string' && config.prompt.trim()) return config.prompt.trim();
|
||||
if (config.promptFile) return (await readFile(path.resolve(REPO_ROOT, config.promptFile), 'utf8')).trim();
|
||||
throw new Error('missing prompt: pass --prompt, --prompt-file, or config.prompt');
|
||||
}
|
||||
|
||||
export function resolveDryRunDesignSystems(config: BatchConfig): string[] {
|
||||
if (config.allDesignSystems) {
|
||||
throw new Error('dry-run with --all-design-systems still requires daemon access; pass explicit --design-systems instead');
|
||||
}
|
||||
const ids = config.designSystems ?? [];
|
||||
if (ids.length === 0) {
|
||||
throw new Error('missing design systems: pass --design-systems, --design-system, --all-design-systems, or config.designSystems');
|
||||
}
|
||||
return dedupeDesignSystemIds(ids);
|
||||
}
|
||||
|
||||
async function resolveDesignSystems(daemonUrl: string, config: BatchConfig): Promise<string[]> {
|
||||
const body = await api<DesignSystemsResponse>(daemonUrl, 'GET', '/api/design-systems');
|
||||
const systems = body.designSystems ?? body.systems ?? [];
|
||||
const availableIds = dedupeDesignSystemIds(systems.map((system) => system.id));
|
||||
if (availableIds.length === 0) throw new Error('daemon returned no design systems');
|
||||
if (config.allDesignSystems) {
|
||||
return availableIds;
|
||||
}
|
||||
const ids = config.designSystems ?? [];
|
||||
if (ids.length === 0) {
|
||||
throw new Error('missing design systems: pass --design-systems, --design-system, --all-design-systems, or config.designSystems');
|
||||
}
|
||||
return validateExplicitDesignSystemIds(ids, availableIds);
|
||||
}
|
||||
|
||||
async function resolveAgentId(daemonUrl: string, config: BatchConfig): Promise<string> {
|
||||
if (typeof config.agentId === 'string' && config.agentId.trim()) return config.agentId.trim();
|
||||
try {
|
||||
const body = await api<AppConfigResponse>(daemonUrl, 'GET', '/api/app-config');
|
||||
const configured = body.config?.agentId;
|
||||
if (typeof configured === 'string' && configured.trim()) return configured.trim();
|
||||
} catch (err) {
|
||||
process.stderr.write(
|
||||
`warning: could not read daemon app config; falling back to ${FALLBACK_AGENT_ID}: ${(err as Error).message || String(err)}\n`,
|
||||
);
|
||||
}
|
||||
return FALLBACK_AGENT_ID;
|
||||
}
|
||||
|
||||
function makeProjectId(designSystemId: string): string {
|
||||
const ts = Date.now().toString(36);
|
||||
const rand = Math.random().toString(36).slice(2, 7);
|
||||
const slug = designSystemId.replace(/[^A-Za-z0-9._-]/g, '-').slice(0, 50);
|
||||
return `${RUN_PREFIX}${slug}-${ts}-${rand}`.slice(0, 128);
|
||||
}
|
||||
|
||||
function makeMessageId(kind: 'user' | 'assistant'): string {
|
||||
return `${RUN_PREFIX}${kind}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 7)}`;
|
||||
}
|
||||
|
||||
function formatBatchTimestamp(date: Date): string {
|
||||
const pad = (value: number) => String(value).padStart(2, '0');
|
||||
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ${pad(date.getHours())}:${pad(date.getMinutes())}`;
|
||||
}
|
||||
|
||||
function terminal(status: RunStatusResponse['status']): boolean {
|
||||
return status === 'succeeded' || status === 'failed' || status === 'canceled';
|
||||
}
|
||||
|
||||
async function waitForRun(daemonUrl: string, runId: string, timeoutMs: number): Promise<RunStatusResponse> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let last: RunStatusResponse | null = null;
|
||||
while (Date.now() < deadline) {
|
||||
last = await api<RunStatusResponse>(daemonUrl, 'GET', `/api/runs/${encodeURIComponent(runId)}`);
|
||||
if (terminal(last.status)) return last;
|
||||
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
||||
}
|
||||
throw createRunTimeoutError(runId, timeoutMs, last?.status);
|
||||
}
|
||||
|
||||
export async function cancelTimedOutRun(daemonUrl: string, error: unknown): Promise<boolean> {
|
||||
if (!isRunTimeoutError(error)) return false;
|
||||
await api<{ ok: true }>(
|
||||
daemonUrl,
|
||||
'POST',
|
||||
`/api/runs/${encodeURIComponent(error.runId)}/cancel`,
|
||||
{},
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function readRunAssistantText(daemonUrl: string, runId: string): Promise<string> {
|
||||
const body = await fetch(`${daemonUrl.replace(/\/$/, '')}/api/runs/${encodeURIComponent(runId)}/events`).then((resp) => {
|
||||
if (!resp.ok) throw new Error(`GET /api/runs/${runId}/events → ${resp.status}`);
|
||||
return resp.text();
|
||||
});
|
||||
return extractAgentTextFromSse(body);
|
||||
}
|
||||
|
||||
async function appendJsonl(output: string | null | undefined, row: BatchResult): Promise<void> {
|
||||
if (!output) return;
|
||||
const absolute = path.resolve(REPO_ROOT, output);
|
||||
await mkdir(path.dirname(absolute), { recursive: true });
|
||||
await writeFile(absolute, `${JSON.stringify(row)}\n`, { flag: 'a' });
|
||||
}
|
||||
|
||||
async function runOne(params: {
|
||||
daemonUrl: string;
|
||||
prompt: string;
|
||||
designSystemId: string;
|
||||
config: Required<Pick<BatchConfig, 'agentId' | 'startRuns' | 'wait' | 'skipDiscoveryBrief' | 'timeoutMs'>> & BatchConfig;
|
||||
}): Promise<BatchResult> {
|
||||
const { daemonUrl, prompt, designSystemId, config } = params;
|
||||
const runPrompt = buildRunPrompt(prompt, config.skipDiscoveryBrief);
|
||||
const projectId = makeProjectId(designSystemId);
|
||||
const projectName = `${config.namePrefix ?? 'DS batch'} — ${designSystemId} — ${formatBatchTimestamp(new Date())}`;
|
||||
const metadata: ProjectMetadata = {
|
||||
kind: DEFAULT_KIND,
|
||||
platform: DEFAULT_PLATFORM,
|
||||
...(config.metadata ?? {}),
|
||||
source: 'batch-design-system-test',
|
||||
batchDesignSystemTest: true,
|
||||
};
|
||||
|
||||
const created = await api<CreateProjectResponse>(daemonUrl, 'POST', '/api/projects', {
|
||||
id: projectId,
|
||||
name: projectName,
|
||||
skillId: config.skillId ?? null,
|
||||
designSystemId,
|
||||
pendingPrompt: runPrompt,
|
||||
metadata,
|
||||
customInstructions: config.customInstructions ?? undefined,
|
||||
skipDiscoveryBrief: config.skipDiscoveryBrief,
|
||||
});
|
||||
const conversationId = created.conversationId ?? null;
|
||||
if (!config.startRuns) {
|
||||
return { designSystemId, projectId, projectName, conversationId, runId: null, status: 'created', daemonUrl };
|
||||
}
|
||||
if (!conversationId) throw new Error('daemon did not return conversationId');
|
||||
|
||||
const now = Date.now();
|
||||
const userMessageId = makeMessageId('user');
|
||||
const assistantMessageId = makeMessageId('assistant');
|
||||
await api(daemonUrl, 'PUT', `/api/projects/${projectId}/conversations/${conversationId}/messages/${userMessageId}`, {
|
||||
role: 'user',
|
||||
content: runPrompt,
|
||||
createdAt: now,
|
||||
});
|
||||
await api(daemonUrl, 'PUT', `/api/projects/${projectId}/conversations/${conversationId}/messages/${assistantMessageId}`, {
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
agentId: config.agentId,
|
||||
agentName: config.agentId,
|
||||
runStatus: 'queued',
|
||||
startedAt: now,
|
||||
createdAt: now,
|
||||
});
|
||||
|
||||
const run = await api<RunCreateResponse>(daemonUrl, 'POST', '/api/runs', {
|
||||
agentId: config.agentId,
|
||||
message: runPrompt,
|
||||
currentPrompt: runPrompt,
|
||||
projectId,
|
||||
conversationId,
|
||||
assistantMessageId,
|
||||
clientRequestId: `${RUN_PREFIX}${projectId}`,
|
||||
skillId: config.skillId ?? null,
|
||||
designSystemId,
|
||||
model: config.model ?? null,
|
||||
reasoning: config.reasoning ?? null,
|
||||
});
|
||||
await api(
|
||||
daemonUrl,
|
||||
'PUT',
|
||||
`/api/projects/${projectId}/conversations/${conversationId}/messages/${assistantMessageId}`,
|
||||
buildAssistantMessageUpdate({
|
||||
agentId: config.agentId,
|
||||
runId: run.runId,
|
||||
runStatus: 'queued',
|
||||
createdAt: now,
|
||||
}),
|
||||
);
|
||||
if (!config.wait) {
|
||||
return { designSystemId, projectId, projectName, conversationId, runId: run.runId, status: 'queued', daemonUrl };
|
||||
}
|
||||
|
||||
let finalStatus: RunStatusResponse;
|
||||
try {
|
||||
finalStatus = await waitForRun(daemonUrl, run.runId, config.timeoutMs);
|
||||
} catch (err) {
|
||||
try {
|
||||
const canceled = await cancelTimedOutRun(daemonUrl, err);
|
||||
if (canceled) {
|
||||
process.stderr.write(`warning: canceled timed-out run ${run.runId} for ${designSystemId}\n`);
|
||||
}
|
||||
} catch (cancelErr) {
|
||||
process.stderr.write(
|
||||
`warning: failed to cancel timed-out run ${run.runId} for ${designSystemId}: ${(cancelErr as Error).message || String(cancelErr)}\n`,
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const assistantText = await readRunAssistantText(daemonUrl, run.runId).catch(() => '');
|
||||
const producedFiles = await api<ProjectFilesResponse>(daemonUrl, 'GET', `/api/projects/${projectId}/files`)
|
||||
.then((body) => body.files ?? [])
|
||||
.catch(() => []);
|
||||
const endedAt = Date.now();
|
||||
await api(
|
||||
daemonUrl,
|
||||
'PUT',
|
||||
`/api/projects/${projectId}/conversations/${conversationId}/messages/${assistantMessageId}`,
|
||||
buildAssistantMessageUpdate({
|
||||
agentId: config.agentId,
|
||||
runId: run.runId,
|
||||
runStatus: finalStatus.status,
|
||||
createdAt: now,
|
||||
content: assistantText,
|
||||
endedAt,
|
||||
producedFiles,
|
||||
}),
|
||||
).catch((err) => {
|
||||
process.stderr.write(`warning: failed to persist assistant message for ${designSystemId}: ${(err as Error).message || String(err)}\n`);
|
||||
});
|
||||
return {
|
||||
designSystemId,
|
||||
projectId,
|
||||
projectName,
|
||||
conversationId,
|
||||
runId: run.runId,
|
||||
status: finalStatus.status,
|
||||
daemonUrl,
|
||||
...(finalStatus.error ? { error: finalStatus.error } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function runWithConcurrency<T, R>(items: T[], concurrency: number, worker: (item: T) => Promise<R>): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
let nextIndex = 0;
|
||||
async function loop(): Promise<void> {
|
||||
for (;;) {
|
||||
const index = nextIndex++;
|
||||
const item = items[index];
|
||||
if (item === undefined) return;
|
||||
results[index] = await worker(item);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => loop()));
|
||||
return results;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const cli = parseArgs(process.argv.slice(2));
|
||||
const fileConfig = await readConfig(cli.configPath);
|
||||
const config = mergeConfig(fileConfig, cli);
|
||||
const prompt = await resolvePrompt(config);
|
||||
const resolved = {
|
||||
...config,
|
||||
agentId: typeof config.agentId === 'string' && config.agentId.trim() ? config.agentId.trim() : FALLBACK_AGENT_ID,
|
||||
startRuns: config.startRuns ?? true,
|
||||
wait: config.wait ?? true,
|
||||
skipDiscoveryBrief: config.skipDiscoveryBrief ?? true,
|
||||
concurrency: config.concurrency ?? 1,
|
||||
timeoutMs: config.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
||||
};
|
||||
|
||||
if (resolved.dryRun) {
|
||||
const designSystems = resolveDryRunDesignSystems(config);
|
||||
const daemonUrl =
|
||||
config.daemonUrl ??
|
||||
process.env.OD_DAEMON_URL ??
|
||||
(isDiscoverablePort(process.env.OD_PORT) ? `http://127.0.0.1:${process.env.OD_PORT}` : '(not resolved in dry-run)');
|
||||
|
||||
console.log(`design-system batch → ${daemonUrl}`);
|
||||
console.log(`prompt: ${prompt.slice(0, 120)}${prompt.length > 120 ? '…' : ''}`);
|
||||
console.log(`design systems (${designSystems.length}): ${designSystems.join(', ')}`);
|
||||
console.log(`agent=${resolved.agentId} skill=${resolved.skillId ?? 'null'} startRuns=${resolved.startRuns} wait=${resolved.wait} concurrency=${resolved.concurrency}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const daemonUrl = await resolveDaemonUrl(config);
|
||||
const designSystems = await resolveDesignSystems(daemonUrl, config);
|
||||
resolved.agentId = await resolveAgentId(daemonUrl, config);
|
||||
|
||||
console.log(`design-system batch → ${daemonUrl}`);
|
||||
console.log(`prompt: ${prompt.slice(0, 120)}${prompt.length > 120 ? '…' : ''}`);
|
||||
console.log(`design systems (${designSystems.length}): ${designSystems.join(', ')}`);
|
||||
console.log(`agent=${resolved.agentId} skill=${resolved.skillId ?? 'null'} startRuns=${resolved.startRuns} wait=${resolved.wait} concurrency=${resolved.concurrency}`);
|
||||
|
||||
if (resolved.output) {
|
||||
const absolute = path.resolve(REPO_ROOT, resolved.output);
|
||||
await mkdir(path.dirname(absolute), { recursive: true });
|
||||
await writeFile(absolute, '');
|
||||
}
|
||||
|
||||
const failures: BatchResult[] = [];
|
||||
const results = await runWithConcurrency(designSystems, resolved.concurrency, async (designSystemId) => {
|
||||
process.stdout.write(` - ${designSystemId}: creating\n`);
|
||||
try {
|
||||
const row = await runOne({ daemonUrl, prompt, designSystemId, config: resolved });
|
||||
await appendJsonl(resolved.output, row);
|
||||
process.stdout.write(` ✓ ${designSystemId}: ${row.status} project=${row.projectId}${row.runId ? ` run=${row.runId}` : ''}\n`);
|
||||
if (row.status === 'failed' || row.status === 'canceled') failures.push(row);
|
||||
return row;
|
||||
} catch (err) {
|
||||
const row: BatchResult = {
|
||||
designSystemId,
|
||||
projectId: '',
|
||||
projectName: '',
|
||||
conversationId: null,
|
||||
runId: null,
|
||||
status: 'failed',
|
||||
daemonUrl,
|
||||
error: (err as Error).message || String(err),
|
||||
};
|
||||
await appendJsonl(resolved.output, row);
|
||||
failures.push(row);
|
||||
process.stderr.write(` ! ${designSystemId}: ${row.error}\n`);
|
||||
return row;
|
||||
}
|
||||
});
|
||||
|
||||
const succeeded = results.filter((r) => r.status === 'created' || r.status === 'queued' || r.status === 'succeeded').length;
|
||||
console.log(`done: ${succeeded}/${results.length} ok${resolved.output ? `; results: ${resolved.output}` : ''}`);
|
||||
if (failures.length > 0) process.exit(1);
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((err) => {
|
||||
console.error(err instanceof Error ? err.message : String(err));
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
/* ─────────────────────────────────────────────────────────────────────────
|
||||
* scripts/check-design-system-flag-parity.ts
|
||||
*
|
||||
* Regression guard for the design-system token channel rolled out in
|
||||
* PR #1385 (the daemon path that injects `tokens.css` + `components.html`
|
||||
* into the agent system prompt when those siblings exist alongside
|
||||
* DESIGN.md). Two brands ship the structured form today (`default`,
|
||||
* `kami`); the other ~138 brands are prose-only and rely on the legacy
|
||||
* DESIGN.md-only behaviour.
|
||||
*
|
||||
* PR-D will flip the `OD_DESIGN_TOKEN_CHANNEL` env default from off to
|
||||
* on. For prose-only brands, that flip MUST be a no-op: missing files
|
||||
* resolve to `undefined`, undefined fields skip the new prompt blocks,
|
||||
* and the composed system prompt should be byte-identical to today's
|
||||
* output. This guard pins that contract end-to-end at every brand:
|
||||
*
|
||||
* 1. checkDesignSystemFlagParity
|
||||
* For each prose-only brand, calling `composeSystemPrompt` with
|
||||
* the new asset fields omitted (flag-off / no files) and with
|
||||
* them set to the values `readDesignSystemAssets` actually
|
||||
* returns (which is `{ undefined, undefined }` for these
|
||||
* brands, i.e. the flag-on path with no files on disk) produces
|
||||
* byte-identical strings.
|
||||
*
|
||||
* For structured brands, the same call pair MUST diverge — the
|
||||
* flag-on path must contain the `tokens.css` and
|
||||
* `components.html` blocks. A silent regression that reverted
|
||||
* the composer to skip those blocks would be caught here.
|
||||
*
|
||||
* The guard mirrors the runtime path the daemon takes in
|
||||
* `apps/daemon/src/server.ts` so the contract is exercised in the
|
||||
* shape that ships, not a synthetic stand-in.
|
||||
*
|
||||
* Run standalone: `pnpm exec tsx scripts/check-design-system-flag-parity.ts`
|
||||
* Or as part of `pnpm guard` (registered in scripts/guard.ts).
|
||||
* ─────────────────────────────────────────────────────────────────── */
|
||||
|
||||
import { readdir, readFile, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
readDesignSystemAssets,
|
||||
type DesignSystemAssets,
|
||||
} from "../apps/daemon/src/design-systems.ts";
|
||||
import { composeSystemPrompt } from "../apps/daemon/src/prompts/system.ts";
|
||||
|
||||
const repoRoot = path.resolve(import.meta.dirname, "..");
|
||||
const designSystemsRoot = path.join(repoRoot, "design-systems");
|
||||
|
||||
const SKIPPED_BRAND_DIRECTORIES = new Set(["_schema"]);
|
||||
|
||||
function toRepositoryPath(filePath: string): string {
|
||||
return path.relative(repoRoot, filePath).split(path.sep).join("/");
|
||||
}
|
||||
|
||||
type BrandSnapshot = {
|
||||
id: string;
|
||||
brandRoot: string;
|
||||
designMdPath: string;
|
||||
designMd: string;
|
||||
title: string;
|
||||
assets: DesignSystemAssets;
|
||||
isStructured: boolean;
|
||||
};
|
||||
|
||||
async function fileExists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
const stats = await stat(filePath);
|
||||
return stats.isFile();
|
||||
} catch (err) {
|
||||
// Only treat genuine "not there" failures as a clean false. Every
|
||||
// other fs error (`EACCES`, `EPERM`, `EIO`, a directory at the
|
||||
// file path, …) means the parity inventory cannot be read — and
|
||||
// since this guard exists precisely to catch silent
|
||||
// misconfigurations during the PR-D rollout, we must surface those
|
||||
// loudly instead of treating them as "brand absent".
|
||||
if (isAbsenceError(err)) return false;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function isAbsenceError(err: unknown): boolean {
|
||||
if (typeof err !== "object" || err === null) return false;
|
||||
const code = (err as { code?: unknown }).code;
|
||||
return code === "ENOENT" || code === "ENOTDIR";
|
||||
}
|
||||
|
||||
function extractTitle(designMd: string, fallback: string): string {
|
||||
const match = /^#\s+(.+?)\s*$/m.exec(designMd);
|
||||
const raw = match?.[1] ?? fallback;
|
||||
// Mirror cleanTitle() in design-systems.ts so the title we feed
|
||||
// composeSystemPrompt matches the daemon's runtime title exactly.
|
||||
return raw.replace(/^Design System (Inspired by|for)\s+/i, "").trim();
|
||||
}
|
||||
|
||||
async function discoverBrandSnapshots(): Promise<BrandSnapshot[]> {
|
||||
let entries;
|
||||
try {
|
||||
entries = await readdir(designSystemsRoot, { withFileTypes: true });
|
||||
} catch (err) {
|
||||
// Same absence-vs-real-error split as `fileExists` above. A
|
||||
// missing `design-systems/` directory is a non-issue (some
|
||||
// packaged distributions ship without it); every other readdir
|
||||
// failure means the parity check cannot enumerate brands and
|
||||
// must fail the guard loudly.
|
||||
if (isAbsenceError(err)) return [];
|
||||
throw err;
|
||||
}
|
||||
|
||||
const snapshots: BrandSnapshot[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory() && !entry.isSymbolicLink()) continue;
|
||||
if (SKIPPED_BRAND_DIRECTORIES.has(entry.name)) continue;
|
||||
|
||||
const id = entry.name;
|
||||
const brandRoot = path.join(designSystemsRoot, id);
|
||||
const designMdPath = path.join(brandRoot, "DESIGN.md");
|
||||
|
||||
if (!(await fileExists(designMdPath))) continue;
|
||||
|
||||
const designMd = await readFile(designMdPath, "utf8");
|
||||
const assets = await readDesignSystemAssets(designSystemsRoot, id);
|
||||
const isStructured =
|
||||
assets.tokensCss !== undefined || assets.fixtureHtml !== undefined;
|
||||
|
||||
snapshots.push({
|
||||
id,
|
||||
brandRoot,
|
||||
designMdPath,
|
||||
designMd,
|
||||
title: extractTitle(designMd, id),
|
||||
assets,
|
||||
isStructured,
|
||||
});
|
||||
}
|
||||
|
||||
snapshots.sort((a, b) => a.id.localeCompare(b.id));
|
||||
return snapshots;
|
||||
}
|
||||
|
||||
function describeFirstDivergence(left: string, right: string): string {
|
||||
const max = Math.min(left.length, right.length);
|
||||
let index = 0;
|
||||
while (index < max && left.charCodeAt(index) === right.charCodeAt(index)) {
|
||||
index += 1;
|
||||
}
|
||||
// Show a small window around the first differing byte so the failure
|
||||
// is actionable without dumping the full ~80 KB system prompt.
|
||||
const window = 60;
|
||||
const start = Math.max(0, index - 8);
|
||||
return [
|
||||
` first divergence at byte ${index} (lengths flag-off=${left.length}, flag-on=${right.length}):`,
|
||||
` flag-off → …${JSON.stringify(left.slice(start, start + window))}…`,
|
||||
` flag-on → …${JSON.stringify(right.slice(start, start + window))}…`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export async function checkDesignSystemFlagParity(): Promise<boolean> {
|
||||
const snapshots = await discoverBrandSnapshots();
|
||||
const violations: string[] = [];
|
||||
|
||||
let proseOnlyChecked = 0;
|
||||
let structuredChecked = 0;
|
||||
|
||||
for (const brand of snapshots) {
|
||||
// Flag-off (today's default): assets fields absent from the
|
||||
// ComposeInput entirely. Equivalent to the legacy code path before
|
||||
// PR #1385 introduced the channel.
|
||||
const flagOff = composeSystemPrompt({
|
||||
designSystemBody: brand.designMd,
|
||||
designSystemTitle: brand.title,
|
||||
});
|
||||
|
||||
// Flag-on: pass exactly what `readDesignSystemAssets` returned at
|
||||
// the top of this loop. For prose-only brands that's
|
||||
// `{ tokensCss: undefined, fixtureHtml: undefined }`, which the
|
||||
// composer already skips; for structured brands it's the verbatim
|
||||
// file contents.
|
||||
const flagOn = composeSystemPrompt({
|
||||
designSystemBody: brand.designMd,
|
||||
designSystemTitle: brand.title,
|
||||
designSystemTokensCss: brand.assets.tokensCss,
|
||||
designSystemFixtureHtml: brand.assets.fixtureHtml,
|
||||
});
|
||||
|
||||
if (brand.isStructured) {
|
||||
structuredChecked += 1;
|
||||
if (flagOn === flagOff) {
|
||||
violations.push(
|
||||
[
|
||||
`[${brand.id}] structured brand produced byte-identical prompts under flag-off vs flag-on — the tokens.css / components.html injection is silently inert.`,
|
||||
` ${toRepositoryPath(brand.brandRoot)} ships ${brand.assets.tokensCss !== undefined ? "tokens.css" : ""}${brand.assets.tokensCss !== undefined && brand.assets.fixtureHtml !== undefined ? " + " : ""}${brand.assets.fixtureHtml !== undefined ? "components.html" : ""} but the composer did not append the corresponding block.`,
|
||||
` This usually means composeSystemPrompt was edited to drop the structured-tier blocks; restore the conditionals around \`designSystemTokensCss\` / \`designSystemFixtureHtml\` in apps/daemon/src/prompts/system.ts.`,
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
proseOnlyChecked += 1;
|
||||
if (flagOn !== flagOff) {
|
||||
violations.push(
|
||||
[
|
||||
`[${brand.id}] prose-only brand produced different prompts under flag-off vs flag-on — flipping OD_DESIGN_TOKEN_CHANNEL would silently change behaviour for this brand.`,
|
||||
describeFirstDivergence(flagOff, flagOn),
|
||||
` Either ${toRepositoryPath(brand.brandRoot)} now ships tokens.css / components.html (in which case it should be promoted to the structured tier and pass the divergence side of this check), or composeSystemPrompt is leaking output for undefined asset fields.`,
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (violations.length > 0) {
|
||||
console.error("Design system flag parity violations:");
|
||||
for (const violation of violations) {
|
||||
console.error(`- ${violation}`);
|
||||
}
|
||||
console.error(
|
||||
"PR-D's promise is that flipping OD_DESIGN_TOKEN_CHANNEL on by default is a no-op for the ~138 prose-only brands and a deliberate uplift for the structured tier. Both halves of that promise must keep holding.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const structuredLabel = `${structuredChecked} structured brand${structuredChecked === 1 ? "" : "s"}`;
|
||||
const proseLabel = `${proseOnlyChecked} prose-only brand${proseOnlyChecked === 1 ? "" : "s"}`;
|
||||
console.log(
|
||||
`Design system flag parity passed: ${proseLabel} produce byte-identical prompts under OD_DESIGN_TOKEN_CHANNEL flag-off vs flag-on; ${structuredLabel} show the expected token-channel divergence.`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
// ─── Standalone entrypoint ───────────────────────────────────────────
|
||||
|
||||
const isInvokedDirectly =
|
||||
process.argv[1] != null && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
|
||||
|
||||
if (isInvokedDirectly) {
|
||||
const passed = await checkDesignSystemFlagParity();
|
||||
if (!passed) process.exitCode = 1;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { readFile, readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { checkDesignSystemFlagParity } from "./check-design-system-flag-parity.ts";
|
||||
import {
|
||||
checkDesignSystemA1RequiredTokens,
|
||||
checkDesignSystemA2DefaultsParity,
|
||||
@@ -708,6 +709,7 @@ const checks: GuardCheck[] = [
|
||||
{ name: "design system B-slot required tokens", run: checkDesignSystemBSlotRequiredTokens },
|
||||
{ name: "design system unknown token allowlist", run: checkDesignSystemUnknownTokens },
|
||||
{ name: "design system A2 defaults parity", run: checkDesignSystemA2DefaultsParity },
|
||||
{ name: "design system flag parity", run: checkDesignSystemFlagParity },
|
||||
];
|
||||
|
||||
const results: boolean[] = [];
|
||||
|
||||
Reference in New Issue
Block a user