External/open-design
1
0
Fork 0
mirror of https://github.com/nexu-io/open-design.git synced 2026-10-27 09:51:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: add AI Native observability trace diagnostics (#3714)

Browse Source 
* docs: spec ai native observability loop

* docs: clarify observability loop gates

Generated-By: looper 0.9.3 (runner=fixer, agent=codex)

* docs: include evaluation runs in score model

Generated-By: looper 0.9.3 (runner=fixer, agent=codex)

* docs: broaden experiment score eligibility

Generated-By: looper 0.9.3 (runner=fixer, agent=codex)

* docs: define observability artifact storage boundary

* docs: expand observability registry rollout

* docs: complete artifact manifest example

Generated-By: looper 0.9.3 (runner=fixer, agent=codex)

* docs: align observability manifest and scoring examples

Generated-By: looper 0.9.3 (runner=fixer, agent=codex)

* docs: define fixed dataset trust gate

* feat: add ai observability trace diagnostics

* docs: clarify object storage issue compatibility

* fix: tighten Langfuse event provenance and attachment caps

Generated-By: looper 0.9.3 (runner=fixer, agent=codex)
This commit is contained in:
yinjialu
2026-06-08 03:43:43 +00:00
committed by GitHub
parent 0252a21e11
commit fcef49b342
9 changed files with 3268 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/daemon/src/claude-stream.ts
+17
View File
@@ -213,6 +213,13 @@ export function createClaudeStreamHandler(onEvent: EventSink) {
if (stopReason) {
onEvent({ type: 'turn_end', stopReason });
}
if (typeof obj.error === 'string' && obj.error.trim()) {
onEvent({
type: 'error',
message: assistantText(obj.message.content) || obj.error,
code: obj.error,
});
}
currentMessageStreamedText = false;
currentMessageStreamedThinking = false;
return;
@@ -247,6 +254,16 @@ export function createClaudeStreamHandler(onEvent: EventSink) {
}
}
function assistantText(content: unknown[]): string {
const parts: string[] = [];
for (const block of content) {
if (isRecord(block) && block.type === 'text' && typeof block.text === 'string') {
parts.push(block.text);
}
}
return parts.join('\n').trim();
}
function handleStreamEvent(ev: Record<string, unknown>) {
if (ev.type === 'message_start') {
// Clean up per-message role-marker guard from the previous message.
apps/daemon/src/langfuse-bridge.ts
+402 -15
View File
@@ -8,6 +8,7 @@
//
// See: specs/change/20260507-langfuse-telemetry/spec.md
import { createHash } from 'node:crypto';
import os from 'node:os';
import { modelIdForTracking } from '@open-design/contracts/analytics';
@@ -19,9 +20,13 @@ import {
readTelemetrySinkConfig,
reportRunCompleted,
reportRunFeedback,
type AgentEventSummary,
type ArtifactManifestEntry,
type ArtifactSummary,
type AttachmentManifestEntry,
type EventsSummary,
type FeedbackReportContext,
type ObjectManifestCompleteness,
type MessageSummary,
type ReportContext,
type RuntimeInfo,
@@ -73,6 +78,12 @@ interface DaemonRunRecord {
promptTelemetry?: PromptStackTelemetry;
}
interface TraceSafeManifestResult {
attachmentManifest: AttachmentManifestEntry[];
artifactManifest: ArtifactManifestEntry[];
completeness: ObjectManifestCompleteness;
}
export interface ReportRunCompletedFromDaemonOpts {
db: unknown;
dataDir: string;
@@ -135,19 +146,27 @@ function summarizeEvents(
events: DaemonRunRecord['events'],
durationMs: number,
): EventsSummary {
let toolCalls = 0;
const toolCallIds = new Set<string>();
let errors = 0;
for (const rec of events) {
const data = rec.data as { type?: string } | null | undefined;
const data = rec.data as { type?: string; id?: unknown } | null | undefined;
if (rec.event === 'agent') {
const t = data?.type;
if (t === 'tool_use') toolCalls += 1;
else if (t === 'error') errors += 1;
if (t === 'tool_use') {
const toolId = data?.id;
if (typeof toolId === 'string' && toolId.length > 0) {
toolCallIds.add(toolId);
} else {
toolCallIds.add(`event-${rec.id}`);
}
} else if (t === 'error') {
errors += 1;
}
} else if (rec.event === 'error') {
errors += 1;
}
}
return { toolCalls, errors, durationMs };
return { toolCalls: toolCallIds.size, errors, durationMs };
}
function messageUsageFromAnalytics(
@@ -209,13 +228,33 @@ function eventTimestamp(
: fallback;
}
function serializeToolPayload(value: unknown): string | undefined {
const CONTENT_TOOL_NAMES = new Set([
'Read',
'Write',
'Edit',
'MultiEdit',
'NotebookEdit',
]);
function redactLocalPaths(value: string): string {
return value
.replace(/\/Users\/[^/\s"']+(?:\/[^ \n\r\t"'`<>)]*)?/g, '[REDACTED:local_path]')
.replace(/[A-Za-z]:\\Users\\[^\\\s"']+(?:\\[^ \n\r\t"'`<>)]*)?/g, '[REDACTED:local_path]');
}
function serializeToolPayload(
value: unknown,
opts: { toolName: string; direction: 'input' | 'output' },
): string | undefined {
if (value === undefined || value === null) return undefined;
if (typeof value === 'string') return redactSecrets(value);
if (CONTENT_TOOL_NAMES.has(opts.toolName)) {
return `[REDACTED:tool_${opts.direction}:content_tool:${opts.toolName}]`;
}
if (typeof value === 'string') return redactLocalPaths(redactSecrets(value));
try {
return redactSecrets(JSON.stringify(value));
return redactLocalPaths(redactSecrets(JSON.stringify(value)));
} catch {
return redactSecrets(String(value));
return redactLocalPaths(redactSecrets(String(value)));
}
}
@@ -247,7 +286,10 @@ function collectToolCalls(
startedAt: timestamp,
endedAt: timestamp,
};
const input = serializeToolPayload(data.input);
const input = serializeToolPayload(data.input, {
toolName: summary.name,
direction: 'input',
});
if (input !== undefined) summary.input = input;
tools.set(data.id, summary);
} else if (
@@ -265,7 +307,10 @@ function collectToolCalls(
endedAt: timestamp,
} satisfies ToolCallSummary);
summary.endedAt = Math.max(summary.startedAt, timestamp);
const output = serializeToolPayload(data.content);
const output = serializeToolPayload(data.content, {
toolName: summary.name,
direction: 'output',
});
if (output !== undefined) summary.output = output;
summary.isError = data.isError === true;
tools.set(data.toolUseId, summary);
@@ -282,15 +327,178 @@ function collectToolCalls(
});
}
function collectAgentEvents(
events: DaemonRunRecord['events'],
runStartedAt: number,
runEndedAt: number,
agentId: string | null | undefined,
): AgentEventSummary[] {
const out: AgentEventSummary[] = [];
const statusCounts = new Map<string, number>();
let thinkingCount = 0;
let usageCount = 0;
const source =
typeof agentId === 'string' && agentId.trim().length > 0
? agentId.trim()
: undefined;
const eventInput = (eventType: string): Record<string, unknown> => ({
...(source ? { source } : {}),
event_type: eventType,
});
for (const rec of events) {
if (rec.event !== 'agent') continue;
const data = rec.data as
| {
type?: string;
label?: unknown;
model?: unknown;
ttftMs?: unknown;
usage?: unknown;
costUsd?: unknown;
durationMs?: unknown;
stopReason?: unknown;
}
| null
| undefined;
const type = data?.type;
const timestamp = Math.min(
Math.max(eventTimestamp(rec, runStartedAt + rec.id), runStartedAt),
runEndedAt,
);
if (type === 'status') {
if (!data) continue;
const label =
typeof data.label === 'string' && data.label.length > 0
? data.label
: 'working';
const index = statusCounts.get(label) ?? 0;
statusCounts.set(label, index + 1);
out.push({
id: `status-${label}-${index}`,
name: `agent-status:${label}`,
timestamp,
input: eventInput('status'),
output: {
label,
...(typeof data.model === 'string' ? { model: data.model } : {}),
...(typeof data.ttftMs === 'number' ? { ttft_ms: data.ttftMs } : {}),
},
});
} else if (type === 'thinking_start') {
const index = thinkingCount;
thinkingCount += 1;
out.push({
id: `thinking-start-${index}`,
name: 'agent-thinking-start',
timestamp,
input: eventInput('thinking_start'),
output: {
status: 'started',
},
});
} else if (type === 'usage') {
if (!data) continue;
const index = usageCount;
usageCount += 1;
out.push({
id: `usage-${index}`,
name: 'agent-usage',
timestamp,
input: eventInput('usage'),
output: {
usage: data.usage,
...(typeof data.costUsd === 'number' ? { cost_usd: data.costUsd } : {}),
...(typeof data.durationMs === 'number'
? { duration_ms: data.durationMs }
: {}),
...(typeof data.stopReason === 'string'
? { stop_reason: data.stopReason }
: {}),
},
});
}
}
return out;
}
function stableObjectId(prefix: 'att' | 'art', parts: unknown[]): string {
const h = createHash('sha256')
.update(JSON.stringify(parts))
.digest('hex')
.slice(0, 16);
return `${prefix}_${h}`;
}
function extensionFromName(value: string): string | undefined {
const basename = value.split(/[\\/]/).pop() ?? '';
const dot = basename.lastIndexOf('.');
if (dot <= 0 || dot === basename.length - 1) return undefined;
return basename.slice(dot + 1).toLowerCase();
}
function safeSha256(value: unknown): string | undefined {
if (typeof value !== 'string') return undefined;
const trimmed = value.trim();
if (!trimmed) return undefined;
return trimmed.startsWith('sha256:') ? trimmed : `sha256:${trimmed}`;
}
function safeStatus(value: unknown): string | undefined {
return typeof value === 'string' && value.trim().length > 0
? value.trim().slice(0, 64)
: undefined;
}
function objectStorageRef(args: {
projectId: string | null | undefined;
runId: string;
objectClass: 'attachment' | 'artifact';
objectId: string;
}): string {
const projectId = typeof args.projectId === 'string' && args.projectId
? args.projectId
: 'unknown-project';
return [
'od://objects',
'workspaces',
'unknown',
'projects',
encodeURIComponent(projectId),
'runs',
encodeURIComponent(args.runId),
args.objectClass,
encodeURIComponent(args.objectId),
].join('/');
}
function statusForSize(size: unknown): {
status: 'ok' | 'partial';
reason?: string;
sizeBytes?: number;
} {
if (typeof size === 'number' && Number.isFinite(size) && size >= 0) {
return { status: 'ok', sizeBytes: Math.floor(size) };
}
return { status: 'partial', reason: 'size_unavailable' };
}
function sanitizeProducedFileSlug(item: Record<string, unknown>): string {
const filePath = typeof item.path === 'string' ? item.path : '';
const name = typeof item.name === 'string' ? item.name : '';
const raw = filePath || name;
if (!raw) return '';
// Keep the legacy field for existing dashboards, but never leak local
// absolute paths through Langfuse.
return raw.split(/[\\/]/).filter(Boolean).pop() ?? raw;
}
function summarizeProducedFiles(items: unknown): ArtifactSummary[] {
if (!Array.isArray(items)) return [];
const out: ArtifactSummary[] = [];
for (const item of items) {
if (!item || typeof item !== 'object' || Array.isArray(item)) continue;
const obj = item as Record<string, unknown>;
const name = typeof obj.name === 'string' ? obj.name : '';
const filePath = typeof obj.path === 'string' ? obj.path : '';
const slug = filePath || name;
const slug = sanitizeProducedFileSlug(obj);
if (!slug) continue;
out.push({
slug,
@@ -301,6 +509,171 @@ function summarizeProducedFiles(items: unknown): ArtifactSummary[] {
return out;
}
function collectPriorUserAttachments(
messages: Array<Record<string, unknown>>,
assistantIndex: number,
): unknown {
const attachments: unknown[] = [];
const priorMessages = messages.slice(
0,
assistantIndex >= 0 ? assistantIndex : messages.length,
);
for (const message of priorMessages) {
if (message.role !== 'user') continue;
const raw = message.attachments;
if (!Array.isArray(raw)) continue;
attachments.push(...raw);
}
return attachments.length > 0 ? attachments : undefined;
}
function buildTraceSafeManifests(args: {
projectId: string | null | undefined;
runId: string;
attachmentsRaw: unknown;
producedFilesRaw: unknown;
}): TraceSafeManifestResult {
const attachmentManifest: AttachmentManifestEntry[] = [];
const artifactManifest: ArtifactManifestEntry[] = [];
let partial = false;
let unavailable = false;
if (Array.isArray(args.attachmentsRaw)) {
for (const [index, raw] of args.attachmentsRaw.entries()) {
const obj = typeof raw === 'string'
? { path: raw, name: raw }
: raw && typeof raw === 'object' && !Array.isArray(raw)
? raw as Record<string, unknown>
: null;
if (!obj) {
partial = true;
continue;
}
const pathValue = typeof obj.path === 'string' ? obj.path : '';
const nameValue = typeof obj.name === 'string' ? obj.name : pathValue;
const attachmentId = stableObjectId('att', [
args.projectId ?? null,
args.runId,
pathValue || nameValue,
index,
]);
const sizeInfo = statusForSize(obj.size);
if (sizeInfo.status === 'partial') partial = true;
const sha256 = safeSha256(obj.sha256 ?? obj.hash);
const extension = extensionFromName(nameValue || pathValue);
attachmentManifest.push({
attachment_id: attachmentId,
object_class: 'attachment',
storage_ref: objectStorageRef({
projectId: args.projectId,
runId: args.runId,
objectClass: 'attachment',
objectId: attachmentId,
}),
status: sizeInfo.status,
...(sizeInfo.reason ? { reason: sizeInfo.reason } : {}),
project_id: args.projectId ?? null,
run_id: args.runId,
workspace_id: null,
...(sizeInfo.sizeBytes !== undefined ? { size_bytes: sizeInfo.sizeBytes } : {}),
...(sha256 ? { sha256 } : {}),
...(typeof obj.mime === 'string' ? { mime_type: obj.mime } : {}),
...(extension ? { extension } : {}),
redacted: false,
truncated: false,
stored_in_open_design: true,
retention_policy: 'project_lifetime',
access_scope: 'project',
sensitivity: 'private',
source: 'user_upload',
expires_at: null,
approved_by: null,
});
}
} else if (args.attachmentsRaw !== undefined && args.attachmentsRaw !== null) {
unavailable = true;
}
if (Array.isArray(args.producedFilesRaw)) {
for (const [index, raw] of args.producedFilesRaw.entries()) {
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
partial = true;
continue;
}
const obj = raw as Record<string, unknown>;
const slug = sanitizeProducedFileSlug(obj);
if (!slug) {
partial = true;
continue;
}
const manifest = obj.artifactManifest && typeof obj.artifactManifest === 'object'
&& !Array.isArray(obj.artifactManifest)
? obj.artifactManifest as Record<string, unknown>
: {};
const artifactId = stableObjectId('art', [
args.projectId ?? null,
args.runId,
slug,
index,
]);
const sizeInfo = statusForSize(obj.size);
if (sizeInfo.status === 'partial') partial = true;
const type =
typeof obj.kind === 'string'
? obj.kind
: typeof manifest.kind === 'string'
? manifest.kind
: 'unknown';
const sha256 = safeSha256(obj.sha256 ?? obj.hash);
const extension = extensionFromName(slug);
const buildStatus = safeStatus(manifest.status);
artifactManifest.push({
artifact_id: artifactId,
object_class: 'artifact',
type,
storage_ref: objectStorageRef({
projectId: args.projectId,
runId: args.runId,
objectClass: 'artifact',
objectId: artifactId,
}),
status: sizeInfo.status,
...(sizeInfo.reason ? { reason: sizeInfo.reason } : {}),
project_id: args.projectId ?? null,
run_id: args.runId,
workspace_id: null,
...(sizeInfo.sizeBytes !== undefined ? { size_bytes: sizeInfo.sizeBytes } : {}),
...(sha256 ? { sha256 } : {}),
...(typeof obj.mime === 'string' ? { mime_type: obj.mime } : {}),
...(extension ? { extension } : {}),
...(typeof manifest.artifactKind === 'string' ? { artifact_kind: manifest.artifactKind } : {}),
...(buildStatus ? { build_status: buildStatus } : {}),
preview_status: 'unavailable',
...(Array.isArray(manifest.exports) && manifest.exports.length > 0
? { export_status: 'available' }
: { export_status: 'unavailable' }),
redacted: false,
truncated: false,
stored_in_open_design: true,
retention_policy: 'project_lifetime',
access_scope: 'project',
sensitivity: 'private',
source: 'agent_generated',
expires_at: null,
approved_by: null,
});
}
} else if (args.producedFilesRaw !== undefined && args.producedFilesRaw !== null) {
unavailable = true;
}
return {
attachmentManifest,
artifactManifest,
completeness: unavailable ? 'unavailable' : partial ? 'partial' : 'complete',
};
}
function pickRunError(
run: DaemonRunRecord,
status: ReportContext['run']['status'],
@@ -341,6 +714,7 @@ export async function reportRunCompletedFromDaemon(
let messageContent = '';
let producedFilesRaw: unknown = undefined;
let attachmentsRaw: unknown = undefined;
if (run.conversationId && run.assistantMessageId) {
try {
// Best-effort. Web persists assistant content via PUT /messages/:id
@@ -349,13 +723,16 @@ export async function reportRunCompletedFromDaemon(
const messages = (
listMessages as (db: unknown, cid: string) => unknown[]
)(db, run.conversationId);
const m = (messages as Array<Record<string, unknown>>).find(
const allMessages = messages as Array<Record<string, unknown>>;
const assistantIndex = allMessages.findIndex(
(x) => x.id === run.assistantMessageId,
);
const m = assistantIndex >= 0 ? allMessages[assistantIndex] : undefined;
if (m) {
messageContent = typeof m.content === 'string' ? m.content : '';
// listMessages returns producedFiles already parsed (db.ts:965).
producedFilesRaw = m.producedFiles;
attachmentsRaw = collectPriorUserAttachments(allMessages, assistantIndex);
}
} catch (err) {
console.warn('[langfuse-bridge] message read failed:', String(err));
@@ -414,6 +791,12 @@ export async function reportRunCompletedFromDaemon(
...getRuntimeInfo(opts.appVersion ?? null),
...(run.clientType ? { clientType: run.clientType } : {}),
};
const manifests = buildTraceSafeManifests({
projectId: run.projectId,
runId: run.id,
attachmentsRaw,
producedFilesRaw,
});
const ctx: ReportContext = {
installationId,
projectId: run.projectId ?? '',
@@ -442,7 +825,11 @@ export async function reportRunCompletedFromDaemon(
...(usage ? { usage } : {}),
},
artifacts: summarizeProducedFiles(producedFilesRaw),
attachmentManifest: manifests.attachmentManifest,
artifactManifest: manifests.artifactManifest,
manifestCompleteness: manifests.completeness,
tools: collectToolCalls(run.events, startedAt, endedAt),
agentEvents: collectAgentEvents(run.events, startedAt, endedAt, run.agentId),
eventsSummary: summarizeEvents(run.events, durationMs),
prefs,
...(turn ? { turn } : {}),
apps/daemon/src/langfuse-trace.ts
+686 -13
View File
@@ -134,6 +134,58 @@ export interface ArtifactSummary {
createdAt?: string;
}
export type ObjectManifestCompleteness = 'complete' | 'partial' | 'unavailable';
export type ObjectManifestStatus = 'ok' | 'partial' | 'unavailable';
export type ObjectManifestSensitivity = 'public' | 'internal' | 'private' | 'sensitive';
export type ObjectManifestAccessScope = 'owner' | 'project' | 'workspace' | 'evaluator';
export type ObjectManifestRetentionPolicy =
| 'ephemeral'
| 'project_lifetime'
| 'eval_fixture'
| 'legal_hold';
export interface TraceSafeObjectManifestBase {
object_class: 'attachment' | 'artifact';
storage_ref: string;
status: ObjectManifestStatus;
reason?: string;
project_id: string | null;
run_id: string;
workspace_id: string | null;
size_bytes?: number;
sha256?: string;
mime_type?: string;
extension?: string;
redacted: boolean;
truncated: boolean;
stored_in_open_design: boolean;
retention_policy: ObjectManifestRetentionPolicy;
access_scope: ObjectManifestAccessScope;
sensitivity: ObjectManifestSensitivity;
source: 'user_upload' | 'agent_generated';
expires_at: string | null;
approved_by: string | null;
}
export interface AttachmentManifestEntry extends TraceSafeObjectManifestBase {
object_class: 'attachment';
attachment_id: string;
}
export interface ArtifactManifestEntry extends TraceSafeObjectManifestBase {
object_class: 'artifact';
artifact_id: string;
type: string;
artifact_kind?: string;
build_status?: string;
preview_status?: string;
export_status?: string;
}
export interface ToolCallSummary {
id: string;
name: string;
@@ -144,6 +196,17 @@ export interface ToolCallSummary {
isError?: boolean;
}
export interface AgentEventSummary {
id: string;
name: string;
timestamp: number;
input?: Record<string, unknown>;
output?: Record<string, unknown>;
metadata?: Record<string, unknown>;
level?: 'DEFAULT' | 'WARNING' | 'ERROR';
statusMessage?: string;
}
export interface EventsSummary {
toolCalls: number;
errors: number;
@@ -188,7 +251,11 @@ export interface ReportContext {
run: RunSummary;
message: MessageSummary;
artifacts: ArtifactSummary[];
attachmentManifest?: AttachmentManifestEntry[];
artifactManifest?: ArtifactManifestEntry[];
manifestCompleteness?: ObjectManifestCompleteness;
tools?: ToolCallSummary[];
agentEvents?: AgentEventSummary[];
eventsSummary: EventsSummary;
prefs: TelemetryPrefs;
langfuse?: LangfuseDeliveryState;
@@ -367,11 +434,14 @@ function timingSpanBody(input: {
name: string;
start: number | undefined;
end: number | undefined;
input?: Record<string, unknown>;
output?: Record<string, unknown>;
metadata?: Record<string, unknown>;
}): Record<string, unknown> | null {
const start = validTimestamp(input.start);
const end = validTimestamp(input.end);
if (start === undefined || end === undefined || end < start) return null;
const durationMs = Math.round(end - start);
return {
id: `${input.runId}-phase-${input.name}`,
traceId: input.traceId,
@@ -379,43 +449,438 @@ function timingSpanBody(input: {
name: input.name,
startTime: new Date(start).toISOString(),
endTime: new Date(end).toISOString(),
input: input.input,
output: {
duration_ms: durationMs,
...(input.output ?? {}),
},
metadata: {
durationMs: Math.round(end - start),
durationMs,
...(input.metadata ?? {}),
},
};
}
function buildTimingSpanBodies(ctx: ReportContext, generationId: string): Record<string, unknown>[] {
function promptBuildSummary(
promptTelemetry: PromptStackTelemetry | undefined,
): Record<string, unknown> {
if (!promptTelemetry) {
return {
prompt_stack_available: false,
};
}
return {
prompt_stack_available: true,
section_count: promptTelemetry.sectionCount,
stack_fingerprint: promptTelemetry.stackFingerprint,
prompt_fingerprint: promptTelemetry.promptFingerprint,
raw_bytes: promptTelemetry.rawBytes,
redacted_bytes: promptTelemetry.redactedBytes,
redacted_content_bytes: promptTelemetry.redactedContentBytes,
};
}
function objectRefSummary(
entries: Array<AttachmentManifestEntry | ArtifactManifestEntry> | undefined,
): Array<Record<string, unknown>> | undefined {
if (!entries?.length) return undefined;
return entries.map((entry) => ({
object_class: entry.object_class,
storage_ref: entry.storage_ref,
status: entry.status,
size_bytes: entry.size_bytes,
sha256: entry.sha256,
mime_type: entry.mime_type,
extension: entry.extension,
redacted: entry.redacted,
truncated: entry.truncated,
retention_policy: entry.retention_policy,
access_scope: entry.access_scope,
sensitivity: entry.sensitivity,
source: entry.source,
...(entry.object_class === 'attachment'
? { attachment_id: entry.attachment_id }
: { artifact_id: entry.artifact_id, type: entry.type }),
}));
}
function cappedManifestEntries<T>(entries: T[] | undefined): T[] | undefined {
return entries ? entries.slice(0, ARTIFACTS_MAX_ITEMS) : undefined;
}
function manifestTruncated(entries: unknown[] | undefined): true | undefined {
return entries && entries.length > ARTIFACTS_MAX_ITEMS ? true : undefined;
}
function tokenUsageSummary(
usage: MessageSummary['usage'],
): Record<string, unknown> | undefined {
if (!usage) return undefined;
return {
input: usage.inputTokens,
input_provider: usage.inputTokensProvider,
input_effective: usage.inputTokensEffective,
output: usage.outputTokens,
total: usage.totalTokens,
cache_read_input: usage.cacheReadInputTokens,
cache_creation_input: usage.cacheCreationInputTokens,
uncached_input: usage.uncachedInputTokens,
cache_hit_ratio: usage.cacheHitRatio,
cache_token_source: usage.cacheTokenSource,
};
}
function latestAgentCostUsd(ctx: ReportContext): number | undefined {
if (!ctx.agentEvents?.length) return undefined;
for (let i = ctx.agentEvents.length - 1; i >= 0; i -= 1) {
const event = ctx.agentEvents[i]!;
const cost = event.output?.cost_usd;
if (typeof cost === 'number' && Number.isFinite(cost) && cost >= 0) {
return cost;
}
}
return undefined;
}
function phaseCost(
phase: string,
costUsd: number | null,
status: string,
source: string,
note?: string,
): Record<string, unknown> {
return {
phase,
cost_usd: costUsd,
cost_status: status,
cost_source: source,
...(note ? { note } : {}),
};
}
function buildCostBreakdown(ctx: ReportContext): Record<string, unknown> {
const costUsd = latestAgentCostUsd(ctx);
const hasCost = costUsd !== undefined;
return {
cost_usd: costUsd ?? null,
currency: 'USD',
pricing_version: hasCost ? 'provider_reported' : 'unavailable',
cost_source: hasCost ? 'agent_usage_event' : 'unavailable',
cost_status: hasCost ? 'available' : 'unavailable',
unavailable_reason: hasCost
? undefined
: 'agent runtime did not report total_cost_usd',
token_usage: tokenUsageSummary(ctx.message.usage),
phase_costs: {
prompt_build: phaseCost(
'prompt-build',
null,
'not_metered',
'not_applicable',
'local prompt assembly; no provider call in this phase',
),
agent_call: phaseCost(
'agent-call',
costUsd ?? null,
hasCost ? 'available' : 'unavailable',
hasCost ? 'agent_usage_event' : 'unavailable',
hasCost
? 'provider-reported total for the agent call; not split across stream/tools/artifact internally'
: 'runtime did not report total_cost_usd',
),
tool_execution: phaseCost(
'tool-execution',
null,
'included_in_agent_call_or_not_metered',
'not_split',
'tool spans are local process/tool time; provider token cost is only available at agent-call granularity',
),
artifact_generation: phaseCost(
'artifact-generation',
null,
'included_in_agent_call',
'not_split',
'artifact output is generated inside the agent call and is not separately priced',
),
verification: phaseCost(
'verification',
null,
'not_instrumented',
'unavailable',
'preview/screenshot/responsive verification is not yet emitted as a structured measured phase',
),
},
};
}
function durationMs(startedAt: number, endedAt: number): number {
return Math.max(0, Math.round(endedAt - startedAt));
}
function buildToolPerformanceDiagnostics(
tools: ToolCallSummary[] | undefined,
): Record<string, unknown> {
const list = tools ?? [];
const byName = new Map<
string,
{
tool_name: string;
call_count: number;
error_count: number;
total_duration_ms: number;
max_duration_ms: number;
min_duration_ms: number;
failure_types: Set<string>;
}
>();
for (const tool of list) {
const d = durationMs(tool.startedAt, tool.endedAt);
const current =
byName.get(tool.name) ??
{
tool_name: tool.name,
call_count: 0,
error_count: 0,
total_duration_ms: 0,
max_duration_ms: 0,
min_duration_ms: Number.POSITIVE_INFINITY,
failure_types: new Set<string>(),
};
current.call_count += 1;
current.total_duration_ms += d;
current.max_duration_ms = Math.max(current.max_duration_ms, d);
current.min_duration_ms = Math.min(current.min_duration_ms, d);
if (tool.isError === true) {
current.error_count += 1;
current.failure_types.add('tool_result_error');
}
byName.set(tool.name, current);
}
return {
tool_call_count: list.length,
total_tool_duration_ms: list.reduce(
(sum, tool) => sum + durationMs(tool.startedAt, tool.endedAt),
0,
),
retry_count_available: false,
retry_count: null,
retry_detection: 'not_instrumented',
retry_unavailable_reason:
'tool spans do not yet carry retry-group or attempt indexes',
by_tool: [...byName.values()].map((entry) => ({
tool_name: entry.tool_name,
call_count: entry.call_count,
error_count: entry.error_count,
total_duration_ms: entry.total_duration_ms,
avg_duration_ms:
entry.call_count > 0
? Math.round(entry.total_duration_ms / entry.call_count)
: 0,
max_duration_ms: entry.max_duration_ms,
min_duration_ms:
Number.isFinite(entry.min_duration_ms) ? entry.min_duration_ms : 0,
retry_count_available: false,
retry_count: null,
failure_types:
entry.failure_types.size > 0 ? [...entry.failure_types] : ['none'],
})),
};
}
function buildArtifactWriteDiagnostics(
ctx: ReportContext,
): Record<string, unknown> {
const writeTools = (ctx.tools ?? []).filter((tool) => tool.name === 'Write');
const totalArtifactSizeBytes = ctx.artifacts.reduce(
(sum, artifact) => sum + artifact.sizeBytes,
0,
);
const writeDurationMs = writeTools.reduce(
(sum, tool) => sum + durationMs(tool.startedAt, tool.endedAt),
0,
);
return {
artifact_count: ctx.artifacts.length,
total_artifact_size_bytes: totalArtifactSizeBytes,
write_tool_count: writeTools.length,
write_tool_duration_ms: writeDurationMs,
bytes_per_write_ms:
writeDurationMs > 0
? Math.round(totalArtifactSizeBytes / writeDurationMs)
: null,
correlation_status:
ctx.artifacts.length > 0 && writeTools.length > 0
? 'heuristic_by_write_tool_total'
: 'unavailable',
correlation_unavailable_reason:
ctx.artifacts.length > 0 && writeTools.length > 0
? undefined
: 'artifact files are not yet linked to individual Write tool ids',
artifacts: ctx.artifacts.map((artifact) => ({
slug: artifact.slug,
type: artifact.type,
size_bytes: artifact.sizeBytes,
})),
};
}
function buildSemanticPhaseDiagnostics(ctx: ReportContext): Record<string, unknown> {
const marks = ctx.run.timingMarks ?? {};
const measured: Record<string, unknown> = {};
const addMeasured = (
name: string,
start: number | undefined,
end: number | undefined,
) => {
const s = validTimestamp(start);
const e = validTimestamp(end);
measured[name] =
s !== undefined && e !== undefined && e >= s
? { duration_ms: Math.round(e - s), status: 'measured' }
: { duration_ms: null, status: 'unmeasured' };
};
addMeasured('prompt-build', marks.promptBuildStartAt, marks.promptBuildEndAt);
addMeasured('agent-call', marks.modelCallStartAt, ctx.run.endedAt);
addMeasured('stream-output', marks.firstTokenAt, marks.finalizeStartAt ?? ctx.run.endedAt);
addMeasured('finalize', marks.finalizeStartAt, ctx.run.endedAt);
return {
measured,
semantic_phase_timing_status: 'partial',
missing_semantic_phases: [
'brief-intake',
'route-task-kind',
'resolve-skill',
'resolve-design-system',
'plan',
'generate-artifact',
'critique',
'repair',
'preview-verify',
'export-finalize',
'evaluator',
],
missing_reason:
'runtime currently emits low-level timing marks but not all product semantic phase boundaries',
};
}
function buildPerformanceDiagnostics(ctx: ReportContext): Record<string, unknown> {
return {
timings: ctx.run.timings,
tool_performance: buildToolPerformanceDiagnostics(ctx.tools),
artifact_write: buildArtifactWriteDiagnostics(ctx),
preview_verify: {
status: 'not_instrumented',
screenshot_check: 'not_reported',
responsive_check: 'not_reported',
html_parse_check: 'not_reported',
note: 'artifact self-checks may appear in assistant output, but are not yet structured observations',
},
semantic_phases: buildSemanticPhaseDiagnostics(ctx),
};
}
function buildTimingSpanBodies(
ctx: ReportContext,
parentObservationId: string,
opts: {
modelCallName?: string;
promptStack?: PromptStackTelemetry;
} = {},
): Record<string, unknown>[] {
const marks = ctx.run.timingMarks ?? {};
const runStart = ctx.run.startedAt;
const runEnd = ctx.run.endedAt;
const queueEnd = marks.promptBuildStartAt ?? marks.startChatRunStartedAt;
const costBreakdown = buildCostBreakdown(ctx);
const phaseCosts = costBreakdown.phase_costs as Record<string, unknown>;
const definitions = [
{
name: 'queue',
start: runStart,
end: queueEnd,
input: {
phase: 'queue',
from: 'run.startedAt',
to: 'promptBuildStartAt',
},
output: {
status: queueEnd === undefined ? 'unmeasured' : 'ready_for_prompt_build',
},
metadata: { boundary: 'run.startedAt -> promptBuildStartAt' },
},
{
name: 'prompt-build',
start: marks.promptBuildStartAt,
end: marks.promptBuildEndAt,
input: {
phase: 'prompt-build',
ingredients: {
agent: ctx.agentId ?? 'unknown',
model: ctx.turn?.model ?? 'unknown',
skill_id: ctx.turn?.skillId ?? null,
design_system_id: ctx.turn?.designSystemId ?? null,
user_request_available: Boolean(ctx.message.prompt),
attachment_refs:
objectRefSummary(cappedManifestEntries(ctx.attachmentManifest)) ?? [],
attachment_refs_truncated: manifestTruncated(ctx.attachmentManifest),
},
},
output: {
status:
marks.promptBuildEndAt === undefined
? 'unmeasured'
: 'prompt_stack_ready',
content_policy: opts.promptStack
? 'redacted_prompt_stack_inline_with_object_refs'
: 'metadata_only_or_unavailable',
...promptBuildSummary(ctx.promptTelemetry),
prompt_stack: opts.promptStack
? structuredPromptStackInput(opts.promptStack)
: undefined,
},
metadata: { boundary: 'promptBuildStartAt -> promptBuildEndAt' },
},
{
name: 'spawn',
start: marks.processSpawnStartedAt,
end: marks.processSpawnedAt,
input: {
phase: 'spawn',
agent: ctx.agentId ?? 'unknown',
runtime: ctx.runtime?.clientType ?? 'unknown',
cwd_ref: 'project',
raw_path_included: false,
},
output: {
status:
marks.processSpawnedAt === undefined ? 'unmeasured' : 'process_spawned',
},
metadata: {
boundary: 'processSpawnStartedAt -> processSpawnedAt',
},
},
{
name: 'model-call',
name: opts.modelCallName ?? 'agent-call',
start: marks.modelCallStartAt,
end: runEnd,
input: {
phase: opts.modelCallName ?? 'agent-call',
model: ctx.turn?.model ?? 'unknown',
agent: ctx.agentId ?? 'unknown',
tool_call_count: ctx.eventsSummary.toolCalls,
generation_observation:
(opts.modelCallName ?? 'agent-call') === 'agent-call',
},
output: {
status: ctx.run.status,
error_code: ctx.run.errorCode,
token_usage: tokenUsageSummary(ctx.message.usage),
cost: phaseCosts.agent_call,
tool_call_count: ctx.eventsSummary.toolCalls,
},
metadata: {
boundary: 'modelCallStartAt -> run.endedAt',
toolCallCount: ctx.eventsSummary.toolCalls,
@@ -425,12 +890,33 @@ function buildTimingSpanBodies(ctx: ReportContext, generationId: string): Record
name: 'stream-output',
start: marks.firstTokenAt,
end: marks.finalizeStartAt ?? runEnd,
input: {
phase: 'stream-output',
from: 'firstTokenAt',
to: 'finalizeStartAt',
},
output: {
status: ctx.run.status,
output_redacted: true,
artifact_blocks_redacted: true,
},
metadata: { boundary: 'firstTokenAt -> finalizeStartAt' },
},
{
name: 'finalize',
start: marks.finalizeStartAt,
end: runEnd,
input: {
phase: 'finalize',
artifact_manifest_enabled: ctx.prefs.artifactManifest === true,
},
output: {
status: ctx.run.status,
artifact_count: ctx.artifacts.length,
attachment_count: ctx.attachmentManifest?.length ?? 0,
manifest_completeness:
ctx.manifestCompleteness ?? (ctx.prefs.artifactManifest ? 'unavailable' : 'off'),
},
metadata: { boundary: 'finalizeStartAt -> run.endedAt' },
},
];
@@ -439,7 +925,7 @@ function buildTimingSpanBodies(ctx: ReportContext, generationId: string): Record
.map((definition) =>
timingSpanBody({
traceId: ctx.run.runId,
parentObservationId: generationId,
parentObservationId,
runId: ctx.run.runId,
...definition,
}),
@@ -447,6 +933,69 @@ function buildTimingSpanBodies(ctx: ReportContext, generationId: string): Record
.filter((body): body is Record<string, unknown> => body !== null);
}
function usageTotal(usage: MessageSummary['usage']): number {
if (!usage) return 0;
const values = [
usage.inputTokens,
usage.inputTokensProvider,
usage.inputTokensEffective,
usage.outputTokens,
usage.totalTokens,
usage.cacheReadInputTokens,
usage.cacheCreationInputTokens,
usage.uncachedInputTokens,
usage.estimatedContextTokens,
];
let total = 0;
for (const value of values) {
if (typeof value === 'number' && Number.isFinite(value)) total += value;
}
return total;
}
function redactArtifactBlocks(value: string | undefined): string | undefined {
if (value === undefined) return undefined;
return value.replace(
/<artifact\b([^>]*)>[\s\S]*?<\/artifact>/gi,
(_match, attrs: string) =>
`<artifact${attrs}>[REDACTED:artifact_content]</artifact>`,
);
}
const CONTENT_TOOL_NAMES = new Set([
'Read',
'Write',
'Edit',
'MultiEdit',
'NotebookEdit',
]);
function redactLocalPaths(value: string | undefined): string | undefined {
if (value === undefined) return undefined;
return value
.replace(/\/Users\/[^/\s"']+(?:\/[^ \n\r\t"'`<>)]*)?/g, '[REDACTED:local_path]')
.replace(/[A-Za-z]:\\Users\\[^\\\s"']+(?:\\[^ \n\r\t"'`<>)]*)?/g, '[REDACTED:local_path]');
}
function traceSafeToolPayload(
toolName: string,
direction: 'input' | 'output',
value: string | undefined,
): string | undefined {
if (value === undefined) return undefined;
if (CONTENT_TOOL_NAMES.has(toolName)) {
return `[REDACTED:tool_${direction}:content_tool:${toolName}]`;
}
return redactLocalPaths(redactArtifactBlocks(value));
}
function shouldCreateGenerationObservation(ctx: ReportContext): boolean {
if (ctx.run.status === 'succeeded') return true;
if (usageTotal(ctx.message.usage) > 0) return true;
if (ctx.eventsSummary.toolCalls > 0) return true;
return ctx.run.failure?.failure_stage !== 'session_init';
}
export function buildTracePayload(ctx: ReportContext): unknown[] {
const wantsContent = ctx.prefs.metrics === true && ctx.prefs.content === true;
const wantsArtifacts = ctx.prefs.artifactManifest === true;
@@ -462,7 +1011,7 @@ export function buildTracePayload(ctx: ReportContext): unknown[] {
? truncate(ctx.message.prompt, INPUT_MAX_BYTES)
: undefined;
const outputText = wantsContent
? truncate(ctx.message.output, OUTPUT_MAX_BYTES)
? truncate(redactArtifactBlocks(ctx.message.output), OUTPUT_MAX_BYTES)
: undefined;
const artifactsList = wantsArtifacts
@@ -472,6 +1021,18 @@ export function buildTracePayload(ctx: ReportContext): unknown[] {
wantsArtifacts && ctx.artifacts.length > ARTIFACTS_MAX_ITEMS
? true
: undefined;
const attachmentManifest = wantsArtifacts
? cappedManifestEntries(ctx.attachmentManifest)
: undefined;
const attachmentManifestTruncated = wantsArtifacts
? manifestTruncated(ctx.attachmentManifest)
: undefined;
const artifactManifest = wantsArtifacts
? cappedManifestEntries(ctx.artifactManifest)
: undefined;
const artifactManifestTruncated = wantsArtifacts
? manifestTruncated(ctx.artifactManifest)
: undefined;
const tokens = ctx.message.usage
? {
@@ -497,6 +1058,8 @@ export function buildTracePayload(ctx: ReportContext): unknown[] {
unit: 'TOKENS' as const,
}
: undefined;
const costBreakdown = buildCostBreakdown(ctx);
const performanceDiagnostics = buildPerformanceDiagnostics(ctx);
const success = ctx.run.status === 'succeeded';
const traceId = ctx.run.runId;
@@ -504,6 +1067,10 @@ export function buildTracePayload(ctx: ReportContext): unknown[] {
ctx.langfuse ?? deriveLangfuseDeliveryState(ctx.prefs, readTelemetrySinkConfig());
const agentSpanId = `${ctx.run.runId}-agent`;
const generationId = `${ctx.run.runId}-gen`;
const createGeneration = shouldCreateGenerationObservation(ctx);
const operationSpanId = createGeneration
? generationId
: `${ctx.run.runId}-runtime`;
const promptStack = ctx.promptTelemetry
? wantsContent
? ctx.promptTelemetry
@@ -532,8 +1099,22 @@ export function buildTracePayload(ctx: ReportContext): unknown[] {
stderr: ctx.run.stderr,
eventsSummary: ctx.eventsSummary,
tokens,
cost_usd: costBreakdown.cost_usd,
currency: costBreakdown.currency,
pricing_version: costBreakdown.pricing_version,
cost_source: costBreakdown.cost_source,
cost_status: costBreakdown.cost_status,
cost_breakdown: costBreakdown,
performance_diagnostics: performanceDiagnostics,
artifacts: artifactsList,
artifactsTruncated,
attachment_manifest: attachmentManifest,
attachment_manifest_truncated: attachmentManifestTruncated,
artifact_manifest: artifactManifest,
artifact_manifest_truncated: artifactManifestTruncated,
manifest_completeness: wantsArtifacts
? (ctx.manifestCompleteness ?? 'unavailable')
: undefined,
projectId: ctx.projectId || undefined,
agent: ctx.agentId,
model: ctx.turn?.model,
@@ -556,12 +1137,16 @@ export function buildTracePayload(ctx: ReportContext): unknown[] {
// shows them in the dedicated Model Parameters card and filters work.
const modelParameters: Record<string, unknown> | undefined =
ctx.turn?.reasoning ? { reasoning: ctx.turn.reasoning } : undefined;
const timingSpanBodies = buildTimingSpanBodies(ctx, generationId);
const timingSpanBodies = buildTimingSpanBodies(ctx, operationSpanId, {
modelCallName: createGeneration ? 'agent-call' : 'runtime-call',
...(promptStack ? { promptStack } : {}),
});
const toolParentObservationId = timingSpanBodies.some(
(span) => span.name === 'model-call',
(span) => span.name === 'agent-call',
)
? `${ctx.run.runId}-phase-model-call`
? `${ctx.run.runId}-phase-agent-call`
: agentSpanId;
const agentEventParentObservationId = toolParentObservationId;
const batch: unknown[] = [
{
@@ -600,10 +1185,16 @@ export function buildTracePayload(ctx: ReportContext): unknown[] {
durationMs: ctx.eventsSummary.durationMs,
toolCalls: ctx.eventsSummary.toolCalls,
errors: ctx.eventsSummary.errors,
cost_usd: costBreakdown.cost_usd,
currency: costBreakdown.currency,
cost_status: costBreakdown.cost_status,
},
},
},
{
];
if (createGeneration) {
batch.push({
id: randomUUID(),
type: 'generation-create',
timestamp: nowIso,
@@ -626,12 +1217,48 @@ export function buildTracePayload(ctx: ReportContext): unknown[] {
usage,
metadata: {
durationMs: ctx.eventsSummary.durationMs,
cost_usd: costBreakdown.cost_usd,
currency: costBreakdown.currency,
pricing_version: costBreakdown.pricing_version,
cost_source: costBreakdown.cost_source,
cost_breakdown: costBreakdown,
performance_diagnostics: performanceDiagnostics,
promptStack,
...promptStackFlatMetadata,
},
},
},
];
});
} else {
batch.push({
id: randomUUID(),
type: 'span-create',
timestamp: nowIso,
body: {
id: operationSpanId,
traceId,
parentObservationId: agentSpanId,
name: 'agent-runtime',
startTime: startTimeIso,
endTime: endTimeIso,
input: generationInput,
output: outputText,
level: 'ERROR',
statusMessage: ctx.run.error ?? undefined,
metadata: {
durationMs: ctx.eventsSummary.durationMs,
cost_usd: costBreakdown.cost_usd,
currency: costBreakdown.currency,
pricing_version: costBreakdown.pricing_version,
cost_source: costBreakdown.cost_source,
cost_breakdown: costBreakdown,
performance_diagnostics: performanceDiagnostics,
promptStack,
...promptStackFlatMetadata,
reason: 'no_model_generation',
},
},
});
}
for (const span of timingSpanBodies) {
batch.push({
@@ -642,16 +1269,45 @@ export function buildTracePayload(ctx: ReportContext): unknown[] {
});
}
if (ctx.agentEvents?.length) {
for (const event of ctx.agentEvents) {
batch.push({
id: randomUUID(),
type: 'event-create',
timestamp: nowIso,
body: {
id: `${ctx.run.runId}-agent-event-${event.id}`,
traceId,
parentObservationId: agentEventParentObservationId,
name: event.name,
startTime: new Date(event.timestamp).toISOString(),
input: event.input,
output: event.output,
level: event.level ?? 'DEFAULT',
statusMessage: event.statusMessage,
metadata: event.metadata,
},
});
}
}
if (ctx.tools?.length) {
for (const tool of ctx.tools) {
const toolSpanId = `${ctx.run.runId}-tool-${tool.id}`;
const toolStartedAt = new Date(tool.startedAt).toISOString();
const toolEndedAt = new Date(tool.endedAt).toISOString();
const toolDurationMs = durationMs(tool.startedAt, tool.endedAt);
const toolInput = wantsContent
? truncate(tool.input, TOOL_INPUT_MAX_BYTES)
? truncate(
traceSafeToolPayload(tool.name, 'input', tool.input),
TOOL_INPUT_MAX_BYTES,
)
: undefined;
const toolOutput = wantsContent
? truncate(tool.output, TOOL_OUTPUT_MAX_BYTES)
? truncate(
traceSafeToolPayload(tool.name, 'output', tool.output),
TOOL_OUTPUT_MAX_BYTES,
)
: undefined;
batch.push({
id: randomUUID(),
@@ -670,9 +1326,13 @@ export function buildTracePayload(ctx: ReportContext): unknown[] {
metadata: {
toolCallId: tool.id,
toolName: tool.name,
durationMs: toolDurationMs,
hasInput: tool.input !== undefined,
hasOutput: tool.output !== undefined,
isError: tool.isError === true,
failureType: tool.isError === true ? 'tool_result_error' : 'none',
retryCount: null,
retryDetection: 'not_instrumented',
},
},
});
@@ -690,9 +1350,22 @@ export function buildTracePayload(ctx: ReportContext): unknown[] {
parentObservationId: agentSpanId,
name: 'artifact-summary',
startTime: endTimeIso,
input: {
source: 'agent_generated_artifacts',
artifact_count: artifactsList.length,
artifact_manifest_enabled: wantsArtifacts,
},
output: {
artifacts: artifactsList,
artifactsTruncated,
manifest_completeness: wantsArtifacts
? (ctx.manifestCompleteness ?? 'unavailable')
: 'off',
},
metadata: {
artifacts: artifactsList,
artifactsTruncated,
artifact_write_diagnostics: performanceDiagnostics.artifact_write,
},
},
});
apps/daemon/src/server.ts
+23
View File
@@ -12957,6 +12957,29 @@ export async function startServer({
if (def.streamFormat === 'claude-stream-json') {
const claude = createClaudeStreamHandler((ev) => {
if (ev?.type === 'error') {
if (agentStreamError) return;
const message = String((ev as any).message || 'Claude Code stream error');
const failureText = [
message,
typeof (ev as any).code === 'string' ? (ev as any).code : '',
agentStdoutTail,
agentStderrTail,
].join('\n');
agentStreamError = rewriteKnownAgentStreamError(
agentId,
message,
failureText,
);
clearInactivityWatchdog();
const serviceCode = classifyAgentServiceFailure(failureText);
send('error', createSseErrorPayload(
serviceCode ?? 'AGENT_EXECUTION_FAILED',
agentStreamError,
{ retryable: serviceCode === 'AGENT_AUTH_REQUIRED' || serviceCode === 'RATE_LIMITED' },
));
return;
}
lastAgentEventPhase = summarizeAgentEventForInactivity(ev);
noteAgentActivity();
noteFirstTokenFromAgentEvent(ev);
apps/daemon/tests/claude-stream-thinking.test.ts
+28
View File
@@ -93,4 +93,32 @@ describe('claude-stream role-marker guard scope', () => {
.join('');
expect(text).toBe('OK.');
});
it('emits an error event when Claude Code marks an assistant message as authentication_failed', () => {
const { events, sink } = collect();
const handler = createClaudeStreamHandler(sink);
handler.feed(JSON.stringify({
type: 'assistant',
error: 'authentication_failed',
message: {
id: 'msg-auth-1',
role: 'assistant',
stop_reason: 'stop_sequence',
content: [
{ type: 'text', text: 'Not logged in · Please run /login' },
],
},
}) + '\n');
expect(events).toContainEqual({
type: 'text_delta',
delta: 'Not logged in · Please run /login',
});
expect(events).toContainEqual({
type: 'error',
message: 'Not logged in · Please run /login',
code: 'authentication_failed',
});
});
});
apps/daemon/tests/langfuse-bridge.test.ts
+445 -4
View File
@@ -10,6 +10,7 @@ interface FakeMessage {
id: string;
role: 'user' | 'assistant';
content: string;
attachments?: Array<Record<string, unknown>>;
producedFiles?: Array<Record<string, unknown>>;
}
@@ -184,12 +185,40 @@ describe('langfuse-bridge.reportRunCompletedFromDaemon', () => {
telemetry: { metrics: true, content: true, artifactManifest: true },
});
const messages: FakeMessage[] = [
{
id: 'user-1',
role: 'user',
content: 'Use this reference.',
attachments: [
{
path: 'uploads/brand.pdf',
name: 'brand.pdf',
kind: 'file',
size: 2_481_032,
mime: 'application/pdf',
sha256: '1234abcd',
},
],
},
{
id: 'msg-1',
role: 'assistant',
content: 'Here is a draft …',
producedFiles: [
{ name: 'index.html', kind: 'html', size: 4096 },
{
name: 'index.html',
path: '/Users/alice/private/project/index.html',
kind: 'html',
mime: 'text/html',
size: 4096,
hash: 'sha256:artifacthash',
artifactManifest: {
version: 1,
kind: 'html',
status: 'complete',
exports: ['html'],
},
},
{ name: 'style.css', kind: 'code', size: 800 },
],
},
@@ -203,7 +232,7 @@ describe('langfuse-bridge.reportRunCompletedFromDaemon', () => {
await reportRunCompletedFromDaemon({
db: makeDbWithListMessages({ 'conv-1': messages }),
dataDir,
run: makeRun() as any,
run: makeRun({ agentId: 'qoder' }) as any,
fetchImpl: fetchSpy as any,
});
} finally {
@@ -217,6 +246,7 @@ describe('langfuse-bridge.reportRunCompletedFromDaemon', () => {
'trace-create',
'span-create',
'generation-create',
'event-create',
'span-create',
'span-create',
'event-create',
@@ -226,6 +256,7 @@ describe('langfuse-bridge.reportRunCompletedFromDaemon', () => {
const generation = bodyOf(batch, 'generation-create', 'llm');
const bash = bodyOf(batch, 'span-create', 'tool:Bash');
const write = bodyOf(batch, 'span-create', 'tool:Write');
const usage = bodyOf(batch, 'event-create', 'agent-usage');
const artifacts = bodyOf(batch, 'event-create', 'artifact-summary');
expect(trace.userId).toBe('install-uuid-1');
expect(trace.sessionId).toBe('conv-1');
@@ -241,17 +272,101 @@ describe('langfuse-bridge.reportRunCompletedFromDaemon', () => {
expect(bash.output).toBe('total 0');
expect(write.parentObservationId).toBe('run-id-1-agent');
expect(write.metadata.toolName).toBe('Write');
expect(usage.parentObservationId).toBe('run-id-1-agent');
expect(usage.input).toEqual({
source: 'qoder',
event_type: 'usage',
});
expect(usage.output.usage).toEqual({ input_tokens: 100, output_tokens: 200 });
expect(artifacts.parentObservationId).toBe('run-id-1-agent');
expect(artifacts.input).toEqual({
source: 'agent_generated_artifacts',
artifact_count: 2,
artifact_manifest_enabled: true,
});
expect(artifacts.output).toEqual({
artifacts: [
{ slug: 'index.html', type: 'html', sizeBytes: 4096 },
{ slug: 'style.css', type: 'code', sizeBytes: 800 },
],
manifest_completeness: 'complete',
});
expect(artifacts.metadata.artifacts).toEqual([
{ slug: 'index.html', type: 'html', sizeBytes: 4096 },
{ slug: 'style.css', type: 'code', sizeBytes: 800 },
]);
expect(trace.metadata.attachment_manifest).toEqual([
expect.objectContaining({
attachment_id: expect.stringMatching(/^att_[0-9a-f]{16}$/),
object_class: 'attachment',
storage_ref: expect.stringMatching(
/^od:\/\/objects\/workspaces\/unknown\/projects\/proj-1\/runs\/run-id-1\/attachment\/att_[0-9a-f]{16}$/,
),
project_id: 'proj-1',
run_id: 'run-id-1',
workspace_id: null,
status: 'ok',
size_bytes: 2_481_032,
sha256: 'sha256:1234abcd',
mime_type: 'application/pdf',
extension: 'pdf',
redacted: false,
truncated: false,
stored_in_open_design: true,
retention_policy: 'project_lifetime',
access_scope: 'project',
sensitivity: 'private',
source: 'user_upload',
expires_at: null,
approved_by: null,
}),
]);
expect(trace.metadata.artifact_manifest).toEqual([
expect.objectContaining({
artifact_id: expect.stringMatching(/^art_[0-9a-f]{16}$/),
object_class: 'artifact',
type: 'html',
storage_ref: expect.stringMatching(
/^od:\/\/objects\/workspaces\/unknown\/projects\/proj-1\/runs\/run-id-1\/artifact\/art_[0-9a-f]{16}$/,
),
project_id: 'proj-1',
run_id: 'run-id-1',
workspace_id: null,
status: 'ok',
size_bytes: 4096,
sha256: 'sha256:artifacthash',
mime_type: 'text/html',
extension: 'html',
build_status: 'complete',
preview_status: 'unavailable',
export_status: 'available',
redacted: false,
truncated: false,
stored_in_open_design: true,
retention_policy: 'project_lifetime',
access_scope: 'project',
sensitivity: 'private',
source: 'agent_generated',
expires_at: null,
approved_by: null,
}),
expect.objectContaining({
object_class: 'artifact',
type: 'code',
size_bytes: 800,
extension: 'css',
export_status: 'unavailable',
}),
]);
expect(trace.metadata.manifest_completeness).toBe('complete');
expect(JSON.stringify(batch)).not.toContain('/Users/alice/private/project');
expect(JSON.stringify(batch)).not.toContain('brand.pdf');
// Core tags must be present. The bridge also tacks on an `os:<...>`
// tag derived from the host (`darwin` / `linux` / `win32`), which is
// useful telemetry but varies between dev / CI environments — assert
// its presence by prefix rather than pinning a value.
expect(trace.tags).toEqual(
expect.arrayContaining(['open-design', 'project:proj-1', 'agent:claude']),
expect.arrayContaining(['open-design', 'project:proj-1', 'agent:qoder']),
);
expect((trace.tags as string[]).some((t) => t.startsWith('os:'))).toBe(true);
expect(trace.metadata.eventsSummary.toolCalls).toBe(2);
@@ -272,6 +387,330 @@ describe('langfuse-bridge.reportRunCompletedFromDaemon', () => {
expect(trace.metadata.success).toBe(true);
});
it('marks trace-safe object manifests partial when object accounting is incomplete', async () => {
await writeAppCfg({
installationId: 'install-uuid-1',
telemetry: { metrics: true, content: true, artifactManifest: true },
});
const fetchSpy = vi
.fn()
.mockResolvedValue(new Response('{}', { status: 207 }));
process.env.LANGFUSE_PUBLIC_KEY = 'pk';
process.env.LANGFUSE_SECRET_KEY = 'sk';
try {
await reportRunCompletedFromDaemon({
db: makeDbWithListMessages({
'conv-1': [
{
id: 'user-1',
role: 'user',
content: 'Use this reference.',
attachments: [{ path: 'uploads/brand.pdf', kind: 'file' }],
},
{
id: 'msg-1',
role: 'assistant',
content: 'Here is a draft …',
producedFiles: [{ name: 'index.html', kind: 'html' }],
},
],
}),
dataDir,
run: makeRun() as any,
fetchImpl: fetchSpy as any,
});
} finally {
delete process.env.LANGFUSE_PUBLIC_KEY;
delete process.env.LANGFUSE_SECRET_KEY;
}
const init = fetchSpy.mock.calls[0]![1] as RequestInit;
const batch = JSON.parse(init.body as string).batch as any[];
const trace = batch[0].body;
expect(trace.metadata.manifest_completeness).toBe('partial');
expect(trace.metadata.attachment_manifest[0]).toMatchObject({
object_class: 'attachment',
status: 'partial',
reason: 'size_unavailable',
});
expect(trace.metadata.artifact_manifest[0]).toMatchObject({
object_class: 'artifact',
status: 'partial',
reason: 'size_unavailable',
});
});
it('carries prior user attachments into follow-up generation traces', async () => {
await writeAppCfg({
installationId: 'install-uuid-1',
telemetry: { metrics: true, content: true, artifactManifest: true },
});
const fetchSpy = vi
.fn()
.mockResolvedValue(new Response('{}', { status: 207 }));
process.env.LANGFUSE_PUBLIC_KEY = 'pk';
process.env.LANGFUSE_SECRET_KEY = 'sk';
try {
await reportRunCompletedFromDaemon({
db: makeDbWithListMessages({
'conv-1': [
{
id: 'user-1',
role: 'user',
content: 'Use this private reference.',
attachments: [
{
path: 'uploads/private-reference.md',
name: 'private-reference.md',
size: 562,
},
],
},
{
id: 'assistant-1',
role: 'assistant',
content: 'Please answer the discovery form.',
},
{
id: 'user-2',
role: 'user',
content: '[form answers — discovery]\nBuild the artifact.',
},
{
id: 'msg-1',
role: 'assistant',
content: 'Done.',
producedFiles: [
{ name: 'index.html', kind: 'html', size: 4096 },
],
},
],
}),
dataDir,
run: (() => {
const now = Date.now();
return makeRun({
analyticsTelemetry: {
promptBuildStartAt: now - 4300,
promptBuildEndAt: now - 4200,
},
promptTelemetry: buildPromptStackTelemetry({
composedPrompt:
'# Instructions\n\nBuild the artifact.\n\n---\n# User request\n\n[form answers — discovery]\nBuild the artifact.',
sections: [
{ kind: 'daemonSystemPrompt', content: 'Build the artifact.' },
{
kind: 'userRequest',
content: '[form answers — discovery]\nBuild the artifact.',
},
],
}),
});
})() as any,
fetchImpl: fetchSpy as any,
});
} finally {
delete process.env.LANGFUSE_PUBLIC_KEY;
delete process.env.LANGFUSE_SECRET_KEY;
}
const init = fetchSpy.mock.calls[0]![1] as RequestInit;
const batch = JSON.parse(init.body as string).batch as any[];
const trace = batch[0].body;
const promptBuild = bodyOf(batch, 'span-create', 'prompt-build');
expect(trace.metadata.attachment_manifest).toHaveLength(1);
expect(trace.metadata.attachment_manifest[0]).toMatchObject({
object_class: 'attachment',
status: 'ok',
size_bytes: 562,
extension: 'md',
source: 'user_upload',
project_id: 'proj-1',
run_id: 'run-id-1',
});
expect(promptBuild.input.ingredients.attachment_refs).toEqual([
expect.objectContaining({
object_class: 'attachment',
status: 'ok',
size_bytes: 562,
extension: 'md',
source: 'user_upload',
attachment_id: trace.metadata.attachment_manifest[0].attachment_id,
}),
]);
const payload = JSON.stringify(batch);
expect(payload).not.toContain('private-reference.md');
expect(payload).not.toContain('uploads/private-reference.md');
});
it('counts duplicate streamed tool_use records by unique tool id', async () => {
await writeAppCfg({
installationId: 'install-uuid-1',
telemetry: { metrics: true, content: true, artifactManifest: false },
});
const fetchSpy = vi
.fn()
.mockResolvedValue(new Response('{}', { status: 207 }));
process.env.LANGFUSE_PUBLIC_KEY = 'pk';
process.env.LANGFUSE_SECRET_KEY = 'sk';
try {
await reportRunCompletedFromDaemon({
db: makeDbWithListMessages({
'conv-1': [
{ id: 'user-1', role: 'user', content: 'Create a file.' },
{ id: 'msg-1', role: 'assistant', content: 'Done.', producedFiles: [] },
],
}),
dataDir,
run: makeRun({
events: [
{
id: 1,
event: 'agent',
timestamp: Date.now() - 3000,
data: {
type: 'tool_use',
id: 'write-1',
name: 'Write',
input: { path: 'index.html', content: '<!doctype html>' },
},
},
{
id: 2,
event: 'agent',
timestamp: Date.now() - 2900,
data: {
type: 'tool_use',
id: 'write-1',
name: 'Write',
input: { path: 'index.html', content: '<!doctype html>' },
},
},
{
id: 3,
event: 'agent',
timestamp: Date.now() - 2500,
data: {
type: 'tool_result',
toolUseId: 'write-1',
content: 'wrote index.html',
isError: false,
},
},
],
}) as any,
fetchImpl: fetchSpy as any,
});
} finally {
delete process.env.LANGFUSE_PUBLIC_KEY;
delete process.env.LANGFUSE_SECRET_KEY;
}
const init = fetchSpy.mock.calls[0]![1] as RequestInit;
const batch = JSON.parse(init.body as string).batch as any[];
const trace = batch[0].body;
const toolSpans = batch.filter(
(item) => item.type === 'span-create' && item.body.name === 'tool:Write',
);
expect(trace.metadata.eventsSummary.toolCalls).toBe(1);
expect(trace.metadata.eventsSummary.errors).toBe(0);
expect(toolSpans).toHaveLength(1);
});
it('redacts content-tool payloads and local paths from tool observations', async () => {
await writeAppCfg({
installationId: 'install-uuid-1',
telemetry: { metrics: true, content: true, artifactManifest: true },
});
const fetchSpy = vi
.fn()
.mockResolvedValue(new Response('{}', { status: 207 }));
process.env.LANGFUSE_PUBLIC_KEY = 'pk';
process.env.LANGFUSE_SECRET_KEY = 'sk';
try {
await reportRunCompletedFromDaemon({
db: makeDbWithListMessages({
'conv-1': [
{
id: 'user-1',
role: 'user',
content: 'Use this private reference.',
attachments: [{ path: 'uploads/private-brand-reference.txt', name: 'private-brand-reference.txt', size: 247 }],
},
{
id: 'msg-1',
role: 'assistant',
content: 'Done.',
producedFiles: [],
},
],
}),
dataDir,
run: makeRun({
events: [
{
id: 1,
event: 'agent',
data: {
type: 'tool_use',
id: 'read-1',
name: 'Read',
input: { file_path: '/Users/alice/project/private-brand-reference.txt' },
},
},
{
id: 2,
event: 'agent',
data: {
type: 'tool_result',
toolUseId: 'read-1',
content: 'Private brand reference: do not upload raw content',
isError: false,
},
},
{
id: 3,
event: 'agent',
data: {
type: 'tool_use',
id: 'write-1',
name: 'Write',
input: { file_path: '/Users/alice/project/index.html', content: '<!doctype html><html>heavy</html>' },
},
},
{
id: 4,
event: 'agent',
data: {
type: 'tool_result',
toolUseId: 'write-1',
content: 'File created successfully at: /Users/alice/project/index.html',
isError: false,
},
},
],
}) as any,
fetchImpl: fetchSpy as any,
});
} finally {
delete process.env.LANGFUSE_PUBLIC_KEY;
delete process.env.LANGFUSE_SECRET_KEY;
}
const init = fetchSpy.mock.calls[0]![1] as RequestInit;
const batch = JSON.parse(init.body as string).batch as any[];
const read = bodyOf(batch, 'span-create', 'tool:Read');
const write = bodyOf(batch, 'span-create', 'tool:Write');
expect(read.input).toBe('[REDACTED:tool_input:content_tool:Read]');
expect(read.output).toBe('[REDACTED:tool_output:content_tool:Read]');
expect(write.input).toBe('[REDACTED:tool_input:content_tool:Write]');
expect(write.output).toBe('[REDACTED:tool_output:content_tool:Write]');
const payload = JSON.stringify(batch);
expect(payload).not.toContain('Private brand reference');
expect(payload).not.toContain('/Users/alice/project');
expect(payload).not.toContain('<!doctype html>');
});
it('forwards run prompt telemetry into trace and generation metadata', async () => {
await writeAppCfg({
installationId: 'install-uuid-1',
@@ -790,7 +1229,9 @@ function makeDbWithListMessages(messagesByConvo: Record<string, FakeMessage[]>)
runStatus: null,
lastRunEventId: null,
eventsJson: null,
attachmentsJson: null,
attachmentsJson: m.attachments
? JSON.stringify(m.attachments)
: null,
commentAttachmentsJson: null,
producedFilesJson: m.producedFiles
? JSON.stringify(m.producedFiles)
apps/daemon/tests/langfuse-trace.test.ts
+698 -3
View File
@@ -297,14 +297,24 @@ describe('buildTracePayload', () => {
const batch = buildTracePayload(
makeCtx({
prefs: { metrics: true, content: true, artifactManifest: false },
message: {
messageId: 'msg-1',
prompt: 'Make a landing page for a coffee shop.',
output:
'Built it.\n<artifact identifier="demo" type="text/html"><!doctype html><html>heavy</html></artifact>',
},
}),
);
const trace = (batch[0] as any).body;
const tool = bodyOf(batch, 'span-create', 'tool:Bash');
const write = bodyOf(batch, 'span-create', 'tool:Write');
expect(trace.input).toMatch(/coffee shop/);
expect(trace.output).toMatch(/landing page draft/);
expect(trace.output).toContain('[REDACTED:artifact_content]');
expect(trace.output).not.toContain('<!doctype html>');
expect(tool.input).toMatch(/ls -la/);
expect(tool.output).toBe('total 0');
expect(write.input).toBe('[REDACTED:tool_input:content_tool:Write]');
expect(write.output).toBe('[REDACTED:tool_output:content_tool:Write]');
});
it('adds prompt-stack metadata to trace and generation without replacing user prompt input', () => {
@@ -434,11 +444,144 @@ describe('buildTracePayload', () => {
{ slug: 'a', type: 'html', sizeBytes: 100 },
{ slug: 'b', type: 'jsx', sizeBytes: 200 },
],
attachmentManifest: [
{
attachment_id: 'att-1',
object_class: 'attachment',
storage_ref: 'od://objects/workspaces/unknown/projects/proj-1/runs/run-1/attachment/att-1',
status: 'ok',
project_id: 'proj-1',
run_id: 'run-1',
workspace_id: null,
size_bytes: 100,
redacted: false,
truncated: false,
stored_in_open_design: true,
retention_policy: 'project_lifetime',
access_scope: 'project',
sensitivity: 'private',
source: 'user_upload',
expires_at: null,
approved_by: null,
},
],
artifactManifest: [
{
artifact_id: 'art-1',
object_class: 'artifact',
type: 'html',
storage_ref: 'od://objects/workspaces/unknown/projects/proj-1/runs/run-1/artifact/art-1',
status: 'ok',
project_id: 'proj-1',
run_id: 'run-1',
workspace_id: null,
size_bytes: 200,
redacted: false,
truncated: false,
stored_in_open_design: true,
retention_policy: 'project_lifetime',
access_scope: 'project',
sensitivity: 'private',
source: 'agent_generated',
expires_at: null,
approved_by: null,
},
],
manifestCompleteness: 'complete',
}),
);
const trace = (batch[0] as any).body;
expect(trace.metadata.artifacts).toBeUndefined();
expect(trace.metadata.artifactsTruncated).toBeUndefined();
expect(trace.metadata.attachment_manifest).toBeUndefined();
expect(trace.metadata.artifact_manifest).toBeUndefined();
expect(trace.metadata.manifest_completeness).toBeUndefined();
});
it('includes trace-safe object manifests when the artifact manifest gate is on', () => {
const batch = buildTracePayload(
makeCtx({
prefs: { metrics: true, content: false, artifactManifest: true },
attachmentManifest: [
{
attachment_id: 'att-1',
object_class: 'attachment',
storage_ref: 'od://objects/workspaces/unknown/projects/proj-1/runs/run-1/attachment/att-1',
status: 'ok',
project_id: 'proj-1',
run_id: 'run-1',
workspace_id: null,
size_bytes: 1024,
sha256: 'sha256:abc',
mime_type: 'application/pdf',
extension: 'pdf',
redacted: false,
truncated: false,
stored_in_open_design: true,
retention_policy: 'project_lifetime',
access_scope: 'project',
sensitivity: 'private',
source: 'user_upload',
expires_at: null,
approved_by: null,
},
],
artifactManifest: [
{
artifact_id: 'art-1',
object_class: 'artifact',
type: 'html',
storage_ref: 'od://objects/workspaces/unknown/projects/proj-1/runs/run-1/artifact/art-1',
status: 'partial',
reason: 'size_unavailable',
project_id: 'proj-1',
run_id: 'run-1',
workspace_id: null,
build_status: 'complete',
preview_status: 'unavailable',
export_status: 'available',
redacted: false,
truncated: false,
stored_in_open_design: true,
retention_policy: 'project_lifetime',
access_scope: 'project',
sensitivity: 'private',
source: 'agent_generated',
expires_at: null,
approved_by: null,
},
],
manifestCompleteness: 'partial',
}),
);
const trace = (batch[0] as any).body;
expect(trace.metadata.attachment_manifest).toEqual([
expect.objectContaining({
attachment_id: 'att-1',
object_class: 'attachment',
storage_ref: expect.stringContaining('/attachment/att-1'),
size_bytes: 1024,
sha256: 'sha256:abc',
retention_policy: 'project_lifetime',
access_scope: 'project',
sensitivity: 'private',
source: 'user_upload',
}),
]);
expect(trace.metadata.artifact_manifest).toEqual([
expect.objectContaining({
artifact_id: 'art-1',
object_class: 'artifact',
type: 'html',
storage_ref: expect.stringContaining('/artifact/art-1'),
status: 'partial',
reason: 'size_unavailable',
build_status: 'complete',
export_status: 'available',
source: 'agent_generated',
}),
]);
expect(trace.metadata.manifest_completeness).toBe('partial');
});
it('caps artifacts at 50 entries with a truncation flag', () => {
@@ -458,6 +601,92 @@ describe('buildTracePayload', () => {
expect(trace.metadata.artifactsTruncated).toBe(true);
});
it('caps artifact manifests at 50 entries with a truncation flag', () => {
const many = Array.from({ length: 75 }, (_, i) => ({
artifact_id: `art-${i}`,
object_class: 'artifact' as const,
type: 'html',
storage_ref: `od://objects/workspaces/unknown/projects/proj-1/runs/run-1/artifact/art-${i}`,
status: 'ok' as const,
project_id: 'proj-1',
run_id: 'run-1',
workspace_id: null,
size_bytes: 1,
redacted: false,
truncated: false,
stored_in_open_design: true,
retention_policy: 'project_lifetime' as const,
access_scope: 'project' as const,
sensitivity: 'private' as const,
source: 'agent_generated' as const,
expires_at: null,
approved_by: null,
}));
const batch = buildTracePayload(
makeCtx({
artifactManifest: many,
manifestCompleteness: 'complete',
prefs: { metrics: true, content: false, artifactManifest: true },
}),
);
const trace = (batch[0] as any).body;
expect(trace.metadata.artifact_manifest).toHaveLength(50);
expect(trace.metadata.artifact_manifest_truncated).toBe(true);
});
it('caps attachment manifests and prompt-build refs at 50 entries', () => {
const many = Array.from({ length: 75 }, (_, i) => ({
attachment_id: `att-${i}`,
object_class: 'attachment' as const,
storage_ref: `od://objects/workspaces/unknown/projects/proj-1/runs/run-1/attachment/att-${i}`,
status: 'ok' as const,
project_id: 'proj-1',
run_id: 'run-1',
workspace_id: null,
size_bytes: i + 1,
sha256: `sha256:att-${i}`,
mime_type: 'application/pdf',
extension: 'pdf',
redacted: false,
truncated: false,
stored_in_open_design: true,
retention_policy: 'project_lifetime' as const,
access_scope: 'project' as const,
sensitivity: 'private' as const,
source: 'user_upload' as const,
expires_at: null,
approved_by: null,
}));
const batch = buildTracePayload(
makeCtx({
attachmentManifest: many,
manifestCompleteness: 'complete',
prefs: { metrics: true, content: false, artifactManifest: true },
run: {
runId: 'run-1',
status: 'succeeded',
startedAt: 1_700_000_000_000,
endedAt: 1_700_000_004_500,
timingMarks: {
promptBuildStartAt: 1_700_000_000_100,
promptBuildEndAt: 1_700_000_000_200,
},
},
}),
);
const trace = (batch[0] as any).body;
const promptBuild = bodyOf(batch, 'span-create', 'prompt-build');
expect(trace.metadata.attachment_manifest).toHaveLength(50);
expect(trace.metadata.attachment_manifest_truncated).toBe(true);
expect(promptBuild.input.ingredients.attachment_refs).toHaveLength(50);
expect(promptBuild.input.ingredients.attachment_refs_truncated).toBe(true);
expect(promptBuild.input.ingredients.attachment_refs.at(-1)).toMatchObject({
attachment_id: 'att-49',
sha256: 'sha256:att-49',
});
});
it('keeps eventsSummary metadata regardless of content / artifact gates', () => {
const batch = buildTracePayload(makeCtx());
const trace = (batch[0] as any).body;
@@ -621,6 +850,67 @@ describe('buildTracePayload', () => {
expect((batch[0] as any).body.metadata.success).toBe(false);
});
it('uses an agent-runtime span instead of an llm generation for session-init failures with no model usage', () => {
const batch = buildTracePayload(
makeCtx({
run: {
runId: 'run-auth',
status: 'failed',
startedAt: 1,
endedAt: 2,
error: 'Not logged in · Please run /login',
errorCode: 'AGENT_AUTH_REQUIRED',
failure: {
failure_category: 'auth',
failure_detail: 'auth_required',
failure_stage: 'session_init',
retryable: false,
user_action: 'login',
},
timingMarks: {
modelCallStartAt: 1,
},
},
message: {
messageId: 'msg-auth',
prompt: 'make an artifact',
output: 'Not logged in · Please run /login',
usage: {
inputTokens: 0,
inputTokensProvider: 0,
inputTokensEffective: 0,
outputTokens: 0,
totalTokens: 0,
cacheReadInputTokens: 0,
cacheCreationInputTokens: 0,
uncachedInputTokens: 0,
estimatedContextTokens: 0,
cacheTokenSource: 'anthropic',
},
},
tools: [],
eventsSummary: { toolCalls: 0, errors: 1, durationMs: 2 },
}),
);
expect(
(batch as Array<{ type: string; body: Record<string, any> }>).find(
(item) => item.type === 'generation-create' && item.body.name === 'llm',
),
).toBeUndefined();
const runtime = bodyOf(batch, 'span-create', 'agent-runtime');
expect(runtime.level).toBe('ERROR');
expect(runtime.statusMessage).toBe('Not logged in · Please run /login');
expect(runtime.metadata.reason).toBe('no_model_generation');
expect(bodyOf(batch, 'span-create', 'runtime-call').parentObservationId).toBe(
'run-auth-runtime',
);
const metadata = (batch[0] as any).body.metadata;
expect(metadata.status).toBe('failed');
expect(metadata.success).toBe(false);
expect(metadata.error_code).toBe('AGENT_AUTH_REQUIRED');
expect(metadata.failure_category).toBe('auth');
});
it('mirrors structured failure fields into trace metadata', () => {
const batch = buildTracePayload(
makeCtx({
@@ -686,8 +976,49 @@ describe('buildTracePayload', () => {
});
it('adds duration spans for run timing marks', () => {
const promptTelemetry = buildPromptStackTelemetry({
composedPrompt:
'# System\n\nUse /Users/alice/project safely\n\n---\n# User request\n\nBuild the card\n\n---\n# Attachments\n\nbrand.pdf',
sections: [
{
kind: 'daemonSystemPrompt',
content: 'Use /Users/alice/project safely',
},
{ kind: 'userRequest', content: 'Build the card' },
{
kind: 'attachments',
metadata: [{ name: 'brand.pdf', size: 1024, mime: 'application/pdf' }],
},
],
});
const batch = buildTracePayload(
makeCtx({
prefs: { metrics: true, content: true, artifactManifest: true },
promptTelemetry,
attachmentManifest: [
{
attachment_id: 'att-1',
object_class: 'attachment',
storage_ref: 'od://objects/workspaces/unknown/projects/proj-1/runs/run-spans/attachment/att-1',
status: 'ok',
project_id: 'proj-1',
run_id: 'run-spans',
workspace_id: null,
size_bytes: 1024,
sha256: 'sha256:attachment',
mime_type: 'application/pdf',
extension: 'pdf',
redacted: false,
truncated: false,
stored_in_open_design: true,
retention_policy: 'project_lifetime',
access_scope: 'project',
sensitivity: 'private',
source: 'user_upload',
expires_at: null,
approved_by: null,
},
],
run: {
runId: 'run-spans',
status: 'succeeded',
@@ -715,24 +1046,388 @@ describe('buildTracePayload', () => {
'queue',
'prompt-build',
'spawn',
'model-call',
'agent-call',
'stream-output',
'finalize',
]),
);
expect(bodyOf(batch, 'span-create', 'prompt-build')).toMatchObject({
input: {
phase: 'prompt-build',
ingredients: {
agent: 'claude',
model: 'unknown',
skill_id: null,
design_system_id: null,
user_request_available: true,
attachment_refs: [
expect.objectContaining({
attachment_id: 'att-1',
storage_ref: expect.stringContaining('/attachment/att-1'),
sha256: 'sha256:attachment',
sensitivity: 'private',
}),
],
},
},
output: {
status: 'prompt_stack_ready',
content_policy: 'redacted_prompt_stack_inline_with_object_refs',
prompt_stack_available: true,
section_count: 3,
prompt_stack: {
type: 'open-design.prompt-stack',
sectionCount: 3,
sections: [
expect.objectContaining({
kind: 'daemonSystemPrompt',
redactedContent: expect.stringContaining('[REDACTED:path]'),
}),
expect.objectContaining({
kind: 'userRequest',
redactedContent: 'Build the card',
}),
expect.objectContaining({
kind: 'attachments',
contentMode: 'metadata-only',
metadata: expect.objectContaining({
count: 1,
}),
}),
],
},
},
});
expect(bodyOf(batch, 'span-create', 'prompt-build').input.prompt_stack).toBeUndefined();
expect(bodyOf(batch, 'span-create', 'spawn')).toMatchObject({
id: 'run-spans-phase-spawn',
parentObservationId: 'run-spans-gen',
input: {
phase: 'spawn',
agent: 'claude',
cwd_ref: 'project',
raw_path_included: false,
},
output: {
duration_ms: 80,
status: 'process_spawned',
},
metadata: {
durationMs: 80,
boundary: 'processSpawnStartedAt -> processSpawnedAt',
},
});
expect(bodyOf(batch, 'span-create', 'agent-call')).toMatchObject({
input: {
phase: 'agent-call',
model: 'unknown',
tool_call_count: 2,
generation_observation: true,
},
output: {
status: 'succeeded',
tool_call_count: 2,
token_usage: {
input: 1234,
input_effective: 1484,
output: 567,
total: 2051,
},
},
});
expect(bodyOf(batch, 'span-create', 'finalize')).toMatchObject({
input: {
phase: 'finalize',
artifact_manifest_enabled: true,
},
output: {
status: 'succeeded',
artifact_count: 0,
attachment_count: 1,
manifest_completeness: 'unavailable',
},
});
expect(bodyOf(batch, 'span-create', 'tool:Bash').parentObservationId).toBe(
'run-spans-phase-model-call',
'run-spans-phase-agent-call',
);
});
it('nests agent status and usage events under agent-call', () => {
const batch = buildTracePayload(
makeCtx({
run: {
runId: 'run-agent-events',
status: 'succeeded',
startedAt: 1_700_000_000_000,
endedAt: 1_700_000_004_500,
timingMarks: {
modelCallStartAt: 1_700_000_000_420,
},
},
agentEvents: [
{
id: 'status-initializing-0',
name: 'agent-status:initializing',
timestamp: 1_700_000_000_500,
input: { source: 'claude-code-stream', event_type: 'status' },
output: { label: 'initializing', model: 'claude-opus-4-8[1m]' },
},
{
id: 'thinking-start-0',
name: 'agent-thinking-start',
timestamp: 1_700_000_000_800,
input: {
source: 'claude-code-stream',
event_type: 'thinking_start',
},
output: { status: 'started' },
},
{
id: 'usage-0',
name: 'agent-usage',
timestamp: 1_700_000_004_000,
input: { source: 'claude-code-stream', event_type: 'usage' },
output: {
usage: { input_tokens: 10, output_tokens: 20 },
cost_usd: 0.01,
stop_reason: 'end_turn',
},
},
],
}),
);
expect(bodyOf(batch, 'event-create', 'agent-status:initializing')).toMatchObject({
parentObservationId: 'run-agent-events-phase-agent-call',
input: {
source: 'claude-code-stream',
event_type: 'status',
},
output: {
label: 'initializing',
model: 'claude-opus-4-8[1m]',
},
});
expect(bodyOf(batch, 'event-create', 'agent-thinking-start')).toMatchObject({
parentObservationId: 'run-agent-events-phase-agent-call',
input: {
source: 'claude-code-stream',
event_type: 'thinking_start',
},
output: { status: 'started' },
});
expect(bodyOf(batch, 'event-create', 'agent-usage')).toMatchObject({
parentObservationId: 'run-agent-events-phase-agent-call',
input: {
source: 'claude-code-stream',
event_type: 'usage',
},
output: {
usage: { input_tokens: 10, output_tokens: 20 },
cost_usd: 0.01,
stop_reason: 'end_turn',
},
});
});
it('emits cost and performance diagnostics for cost governance', () => {
const batch = buildTracePayload(
makeCtx({
prefs: { metrics: true, content: true, artifactManifest: true },
artifacts: [
{ slug: 'index.html', type: 'html', sizeBytes: 4096 },
{ slug: 'brand-spec.md', type: 'text', sizeBytes: 1024 },
],
run: {
runId: 'run-cost-perf',
status: 'succeeded',
startedAt: 1_700_000_000_000,
endedAt: 1_700_000_006_000,
timings: {
generation_duration_ms: 5000,
tool_call_count: 2,
tool_duration_ms: 1700,
total_duration_ms: 6000,
},
timingMarks: {
promptBuildStartAt: 1_700_000_000_100,
promptBuildEndAt: 1_700_000_000_200,
modelCallStartAt: 1_700_000_000_300,
firstTokenAt: 1_700_000_001_000,
finalizeStartAt: 1_700_000_005_500,
},
},
agentEvents: [
{
id: 'usage-0',
name: 'agent-usage',
timestamp: 1_700_000_005_400,
input: { source: 'claude-code-stream', event_type: 'usage' },
output: {
usage: { input_tokens: 100, output_tokens: 200 },
cost_usd: 0.1234,
duration_ms: 5400,
},
},
],
}),
);
const trace = bodyOf(batch, 'trace-create');
const generation = bodyOf(batch, 'generation-create', 'llm');
const agentCall = bodyOf(batch, 'span-create', 'agent-call');
const bash = bodyOf(batch, 'span-create', 'tool:Bash');
const write = bodyOf(batch, 'span-create', 'tool:Write');
const artifacts = bodyOf(batch, 'event-create', 'artifact-summary');
expect(trace.metadata).toMatchObject({
cost_usd: 0.1234,
currency: 'USD',
pricing_version: 'provider_reported',
cost_source: 'agent_usage_event',
cost_status: 'available',
cost_breakdown: {
cost_usd: 0.1234,
currency: 'USD',
phase_costs: {
prompt_build: {
phase: 'prompt-build',
cost_usd: null,
cost_status: 'not_metered',
},
agent_call: {
phase: 'agent-call',
cost_usd: 0.1234,
cost_status: 'available',
},
artifact_generation: {
phase: 'artifact-generation',
cost_status: 'included_in_agent_call',
},
verification: {
phase: 'verification',
cost_status: 'not_instrumented',
},
},
},
performance_diagnostics: {
tool_performance: {
tool_call_count: 2,
total_tool_duration_ms: 1700,
retry_count_available: false,
retry_count: null,
by_tool: expect.arrayContaining([
expect.objectContaining({
tool_name: 'Bash',
call_count: 1,
total_duration_ms: 800,
failure_types: ['none'],
}),
expect.objectContaining({
tool_name: 'Write',
call_count: 1,
total_duration_ms: 900,
failure_types: ['none'],
}),
]),
},
artifact_write: {
artifact_count: 2,
total_artifact_size_bytes: 5120,
write_tool_count: 1,
write_tool_duration_ms: 900,
correlation_status: 'heuristic_by_write_tool_total',
},
preview_verify: {
status: 'not_instrumented',
screenshot_check: 'not_reported',
responsive_check: 'not_reported',
},
semantic_phases: {
semantic_phase_timing_status: 'partial',
missing_semantic_phases: expect.arrayContaining([
'route-task-kind',
'preview-verify',
'evaluator',
]),
},
},
});
expect(generation.metadata.cost_usd).toBe(0.1234);
expect(generation.metadata.performance_diagnostics.preview_verify.status).toBe(
'not_instrumented',
);
expect(agentCall.output.cost).toMatchObject({
phase: 'agent-call',
cost_usd: 0.1234,
cost_status: 'available',
});
expect(bash.metadata).toMatchObject({
durationMs: 800,
failureType: 'none',
retryCount: null,
retryDetection: 'not_instrumented',
});
expect(write.metadata).toMatchObject({
durationMs: 900,
failureType: 'none',
});
expect(artifacts.metadata.artifact_write_diagnostics).toMatchObject({
total_artifact_size_bytes: 5120,
write_tool_duration_ms: 900,
});
});
it('marks cost unavailable when the runtime does not report provider cost', () => {
const batch = buildTracePayload(makeCtx());
const trace = bodyOf(batch, 'trace-create');
expect(trace.metadata).toMatchObject({
cost_usd: null,
currency: 'USD',
pricing_version: 'unavailable',
cost_source: 'unavailable',
cost_status: 'unavailable',
cost_breakdown: {
unavailable_reason: 'agent runtime did not report total_cost_usd',
phase_costs: {
agent_call: {
cost_usd: null,
cost_status: 'unavailable',
},
},
},
});
});
it('keeps prompt-build ingredient keys stable when optional inputs are absent', () => {
const batch = buildTracePayload(
makeCtx({
run: {
runId: 'run-prompt-ingredients',
status: 'succeeded',
startedAt: 1_700_000_000_000,
endedAt: 1_700_000_001_000,
timingMarks: {
promptBuildStartAt: 1_700_000_000_100,
promptBuildEndAt: 1_700_000_000_200,
},
},
}),
);
expect(bodyOf(batch, 'span-create', 'prompt-build').input).toMatchObject({
phase: 'prompt-build',
ingredients: {
agent: 'claude',
model: 'unknown',
skill_id: null,
design_system_id: null,
user_request_available: true,
attachment_refs: [],
},
});
});
it('passes through anonymous installationId as userId', () => {
const batch = buildTracePayload(makeCtx({ installationId: null }));
expect((batch[0] as any).body.userId).toBeUndefined();
docs/ai-native-observability-trace-analysis.md
+380
View File
@@ -0,0 +1,380 @@
# AI Native Observability Trace Analysis Guide
This document records how to read the new Open Design Langfuse trace shape for
AI Native observability. It is intended for product, engineering, and Agent
reviewers who need to decide whether an Agent candidate improves quality,
stability, latency, and cost without weakening user trust.
Related links:
- PR: https://github.com/nexu-io/open-design/pull/3714
- Issue: https://github.com/nexu-io/open-design/issues/3733
- Full UI validation, alignment turn: https://us.cloud.langfuse.com/project/cmov9fhj905kaad06h8to8rqu/traces/f486abe4-f45d-4af2-b0c2-8b8675140c78
- Full UI validation, artifact turn: https://us.cloud.langfuse.com/project/cmov9fhj905kaad06h8to8rqu/traces/e0614ca6-bbe8-41a4-8022-ccab09a66efa
## Review Goal
The trace should help a human reviewer answer four questions:
1. Did the Agent produce the expected artifact and complete the task?
2. Did quality and stability pass the fixed dataset and evaluator gates?
3. Did latency and cost improve or at least not regress?
4. Are attachments and artifacts handled in a way that preserves user trust?
The intended decision rule is:
```text
PASS only when:
- quality does not regress on the fixed dataset,
- stability does not regress,
- cost and latency improve or remain within the agreed threshold,
- sensitive attachments/artifacts are only promoted to long-term fixtures after human approval.
```
## Trace Reading Order
Start at the root trace metadata, then drill into these observations:
1. `prompt-build`
2. `agent-call`
3. `tool:*`
4. `artifact-summary` when present, otherwise root `performance_diagnostics.artifact_write`
5. `agent-usage`
6. `finalize`
This order follows the human review flow: understand what prompt was sent,
inspect the Agent call, attribute tool and artifact cost/performance, then
confirm final status.
## Root Trace Metadata
Use root metadata for the release summary:
- `status` / `success`: final run result.
- `cost_usd`: provider-reported total cost when available.
- `currency`: expected `USD`.
- `pricing_version`: currently `provider_reported` when the runtime emits cost.
- `cost_source`: currently `agent_usage_event`.
- `cost_breakdown`: cost split by review phase.
- `performance_diagnostics`: consolidated performance and instrumentation view.
- `attachment_manifest`: trace-safe attachment references.
- `artifact_manifest`: trace-safe generated artifact references.
The root trace is the right place for dashboards and pass/fail summaries because
it stays available even when a reviewer does not expand individual observations.
## Prompt Build
`prompt-build.input` records the construction materials:
- `phase`: `prompt-build`.
- `ingredients.agent`
- `ingredients.model`
- `ingredients.skill_id`
- `ingredients.design_system_id`
- `ingredients.user_request_available`
- `ingredients.attachment_refs`
`prompt-build.output` records the resulting prompt stack:
- `status`: expected `prompt_stack_ready`.
- `content_policy`: expected `redacted_prompt_stack_inline_with_object_refs`.
- `section_count`
- `stack_fingerprint`
- `prompt_fingerprint`
- `raw_bytes`
- `redacted_bytes`
- `redacted_content_bytes`
- `prompt_stack.sections`
The section list is enough to analyze prompt composition, size, truncation, and
fingerprint changes. It is not a full raw prompt archive. Heavy or sensitive
content should remain in Open Design object storage and be referenced by
`storage_ref`, hash, and retention policy rather than inlining full raw prompt
text into Langfuse.
Current validation showed 7 sections. That is expected for the full UI path:
- first-turn trace: system, runtime tools, echo guard, user request, plugin
stage prompt, cwd hint, attachments.
- follow-up trace: form override, system, runtime tools, echo guard, answered
user request, plugin stage prompt, cwd hint.
## Agent Call
`agent-call` is the runtime phase that contains the provider call and streamed
Agent execution. Use:
- `agent-call.input.model`
- `agent-call.input.agent`
- `agent-call.input.tool_call_count`
- `agent-call.output.duration_ms`
- `agent-call.output.status`
- `agent-call.output.token_usage`
- `agent-call.output.cost`
`agent-call.output.cost` is currently the best source for provider cost at run
granularity. It is not yet a precise internal split across tool execution,
artifact generation, and verification. The trace makes this explicit in
`cost_breakdown.phase_costs`.
## Cost Governance
Use `cost_breakdown.phase_costs` to read cost status:
- `prompt_build`: `not_metered`; local prompt assembly.
- `agent_call`: provider-reported cost when available.
- `tool_execution`: included in agent call or local process time; not split.
- `artifact_generation`: included in agent call; not separately priced.
- `verification`: currently `not_instrumented`.
This structure is intentionally explicit. It avoids pretending we have precise
phase cost attribution before the runtime can emit it.
For long-term governance, dashboards should aggregate:
- total `cost_usd` by model, task type, skill, and design system;
- cache hit ratio from `token_usage`;
- cost per successful artifact;
- cost per approved dataset case;
- cost deltas between experiment candidate and baseline.
## Tool Performance
Use `performance_diagnostics.tool_performance` and each `tool:*` observation.
Root aggregate fields:
- `tool_call_count`
- `total_tool_duration_ms`
- `by_tool[].tool_name`
- `by_tool[].call_count`
- `by_tool[].error_count`
- `by_tool[].avg_duration_ms`
- `by_tool[].max_duration_ms`
- `by_tool[].min_duration_ms`
- `by_tool[].failure_types`
- `retry_detection`
Per tool observation metadata:
- `toolName`
- `durationMs`
- `hasInput`
- `hasOutput`
- `isError`
- `failureType`
- `retryCount`
- `retryDetection`
Current limitation: retry count is marked `not_instrumented` because tool spans
do not yet carry retry group or attempt indexes. This is a known follow-up for
stable retry governance.
## Artifact Analysis
Use `artifact-summary` when present. If the observation is absent, use root
`performance_diagnostics.artifact_write`.
Important fields:
- `artifact_count`
- `total_artifact_size_bytes`
- `write_tool_count`
- `write_tool_duration_ms`
- `bytes_per_write_ms`
- `correlation_status`
- `artifacts[].slug`
- `artifacts[].type`
- `artifacts[].size_bytes`
The current correlation is heuristic: artifact files are linked to total Write
tool duration, not individual file-level write spans. This is useful for
spotting large write outputs but not enough for per-file performance attribution.
Current validation note: the artifact turn reported artifact write diagnostics
on the root trace, but Langfuse did not show a separate `artifact-summary`
observation in the observations API response. Product reviewers can still read
artifact size and write-duration correlation from root metadata, but the trace
tree should be improved so `artifact-summary` is reliably visible.
## Preview Verification
Use `performance_diagnostics.preview_verify`.
Current expected values:
- `status`: `not_instrumented`
- `screenshot_check`: `not_reported`
- `responsive_check`: `not_reported`
- `html_parse_check`: `not_reported`
This is a deliberate honesty field. If preview, screenshot, responsive, or
console-error checks are not emitted as structured observations, the trace must
not imply that visual verification has passed.
Release gate implication: visual artifacts should not be marked fully trusted
until preview verification emits structured pass/fail evidence.
## Semantic Phase Timing
Use `performance_diagnostics.semantic_phases`.
Measured today:
- `prompt-build`
- `agent-call`
- `stream-output`
- `finalize`
Known missing semantic phases:
- `brief-intake`
- `route-task-kind`
- `resolve-skill`
- `resolve-design-system`
- `plan`
- `generate-artifact`
- `critique`
- `repair`
- `preview-verify`
- `export-finalize`
- `evaluator`
This means the trace can explain runtime performance, but not yet every product
semantic phase. For AI Native observability, these semantic phases should become
first-class observations so product and engineering can identify where latency
or quality regressions originate.
## Dataset, Evaluator, And Experiment Gates
Trace data alone should not be the final release gate. The trust loop requires
fixed datasets, evaluators, and experiments:
- Datasets define stable fixture cases.
- Evaluators compute pass/fail and score evidence.
- Experiments compare candidate and baseline versions on the same dataset.
Minimum pass/fail rule:
```text
Candidate may ship only if:
- fixed dataset quality is not worse than baseline,
- stability is not worse than baseline,
- cost and latency improve or remain within threshold,
- sensitive attachment/artifact cases have human approval before long-term fixture promotion.
```
Trace should link to dataset item ids, evaluator ids, experiment ids, baseline
run ids, and candidate run ids when those features are wired.
## Attachment And Artifact Trust
For user attachments and generated artifacts, Langfuse should contain trace-safe
references:
- `storage_ref`
- `size_bytes`
- `sha256` when available
- `summary` when available
- `retention_policy`
- `access_scope`
- `sensitivity`
- `approved_by`
Raw sensitive content belongs in Open Design object storage with access control,
retention policy, and deletion semantics. Langfuse should remain the
observability surface, not the source-of-truth storage surface for private
customer materials.
## Issue #3733 Compatibility Check
Issue #3733 targets the first durable bridge between Open Design-owned object
references and Langfuse. Its current implementation boundary is intentionally
narrow: add trace-safe attachment/artifact manifests that can resolve back to
Open Design-owned project storage today and to a ProjectStorage or
S3-compatible backend later. It does not require full object-storage
productization, ACL enforcement, quota enforcement, signed download URLs, or
lifecycle cleanup in the same slice.
This branch is compatible with that goal. The current implementation already
uses Langfuse-safe manifest fields and `od://objects/...` references rather than
raw attachment bodies, generated artifact bodies, local absolute paths, or
client-usable URLs. That means Langfuse can show which private inputs and
outputs were involved in a run without becoming the raw content store.
The important scope boundary is:
| Area | Current branch | Issue #3733 target | Conflict check |
| --- | --- | --- | --- |
| Langfuse trace metadata | Adds attachment/artifact manifests, cost diagnostics, tool diagnostics, and prompt stack metadata. | Requires trace-safe manifest metadata behind telemetry gates. | No conflict; current branch implements part of the target. |
| Storage reference | Emits stable `od://objects/...` references. | Treats `storage_ref` as an Open Design registry reference, future-compatible with ProjectStorage/S3 backends. | No conflict; current refs should remain non-dereferenceable from Langfuse. |
| Raw content handling | Keeps attachments and artifact bodies out of Langfuse. | Requires privacy tests proving raw bodies, local paths, signed URLs, and credentials are absent. | No conflict; this is the intended privacy boundary. |
| Actual object storage backend | Not implemented here. Existing project storage remains the source used by the manifest builder. | Full object storage productization is out of scope for #3733, but future-compatible schema is required. | No conflict; this is a remaining backend follow-up, not a blocker. |
| Governance/accounting fields | Adds retention, access, sensitivity, source, run/project/workspace attribution where available. | Requires defaults for governance and future quota/cost attribution. | No conflict; current branch aligns with the schema direction. |
| Dataset promotion | Documents that sensitive cases require human approval before long-term fixture promotion. | Needs manifest data reusable by future fixed-dataset trust gates. | No conflict; implementation still needs dataset/evaluator/experiment links later. |
Remaining work for #3733 after this branch:
1. Make the manifest builder resolve through a concrete Open Design object
registry boundary instead of only best-effort project/run surfaces.
2. Fill `workspace_id` and `sha256` consistently for every attachment and
artifact when the backing storage can provide them.
3. Add explicit tests that generated Langfuse batches do not contain raw
attachment text, artifact body content, local absolute paths, credentials,
signed URLs, or client-usable upload/download URLs.
4. Preserve `storage_ref` as a stable, non-secret reference and require any
future download or preview access to go through Open Design-controlled
authorization.
5. Decide the first real backend path: local ProjectStorage-compatible registry
first, then optional S3-compatible object storage once quota, retention, and
deletion semantics are designed.
## Validation Result
Validation was performed after merging the latest `origin/main` into the branch.
Branch state:
- current branch: `codex/ai-native-observability-loop-spec`
- `origin/main` is an ancestor of the current branch
- ahead/behind against `origin/main`: `10/0` before committing this document
Local checks:
- `pnpm install`
- `pnpm --filter @open-design/daemon exec vitest run -c vitest.config.ts tests/langfuse-trace.test.ts tests/langfuse-bridge.test.ts tests/claude-stream-thinking.test.ts`
- `pnpm --filter @open-design/daemon typecheck`
- `pnpm guard`
End-to-end validation:
- UI onboarding completed on an isolated data directory.
- Real Claude Code runtime selected.
- User attachment uploaded through the UI.
- First turn emitted an alignment form.
- Human answers were submitted through the UI.
- Second turn generated a live HTML artifact.
- Langfuse traces were queried through the public API.
Validated trace results:
- first alignment trace succeeded with cost and prompt-build metadata;
- artifact trace succeeded with provider cost, prompt stack, tool diagnostics,
artifact write diagnostics, and explicit preview/semantic instrumentation gaps;
- no `tool:*` observation had undefined input or output;
- attachment references were object refs, not raw attachment content.
## Current Gaps To Track
1. Store complete raw/redacted prompt payloads in Open Design object storage and
add `full_prompt_ref`, `full_prompt_sha256`, `full_prompt_bytes`, and
`full_prompt_content_policy` to `prompt-build.output`.
2. Emit retry group and attempt indexes for tool calls.
3. Link individual artifact files to individual write spans.
4. Emit structured preview verification observations for screenshot,
responsive, console-error, and HTML-parse checks.
5. Promote product semantic phases to first-class observations.
6. Link traces to Langfuse dataset item, evaluator, and experiment ids.
7. Make `artifact-summary` reliably visible as a Langfuse observation whenever
artifacts are present.
specs/current/ai-native-observability-loop.md
+589
View File
@@ -0,0 +1,589 @@
# AI Native Observability Loop
## Purpose
Define the target observability loop that lets Open Design improve agent
quality, reliability, latency, and cost from production evidence.
This is a planning spec. It does not implement runtime behavior. It starts from
the existing Langfuse trace forwarding and PostHog run analytics, then defines
the target loop that adds datasets, experiments, annotation, and release gates
around those signals.
Issue: [#3713](https://github.com/nexu-io/open-design/issues/3713)
## Current State
Open Design already reports completed runs to Langfuse and PostHog. The current
implementation captures useful operational facts:
- trace identity: `run_id == langfuse_trace_id == traceId`;
- run status, error code, failure category, failure detail, retryability, and
user action;
- timing fields for queue, prompt build, spawn, first token, generation, tool
aggregate, finalize, and total duration;
- token and cache fields for provider input, effective input, cache read/write,
uncached input, estimated context tokens, and cache source;
- tags and metadata for agent, model, skill, design system, runtime, app
channel, operating system, and client type;
- user feedback scores such as `user_rating` and `user_rating_reason`.
The current Langfuse tree is still mostly runtime-oriented. A typical trace is
shaped like:
```text
open-design-turn
agent-run
llm
tool:read
tool:write
tool:todowrite
```
This answers "what did the subprocess and tools do?" It does not consistently
answer "which product task stage made the run succeed, fail, become slow, become
expensive, or require human judgment?"
The current project also has far more traces than durable evaluation assets.
That means Open Design can inspect individual runs, but the system does not yet
turn production evidence into enough datasets, experiments, annotation queues,
and release gates to drive continuous improvement.
The storage foundation is partly in place but not yet connected to this
observability boundary:
- project files, uploads, pasted content, generated files, and live artifacts
currently live under the local project filesystem rooted at
`<dataDir>/projects/<projectId>`;
- saved standalone HTML artifacts live under `<dataDir>/artifacts` and are
served through the daemon's static `/artifacts/*` route;
- media generation writes its output into the same project file directory;
- `ProjectStorage`, `LocalProjectStorage`, and `S3ProjectStorage` already define
a read/write/list/delete/stat adapter contract for S3-compatible stores, but
the current project file and artifact routes still call the local filesystem
helpers directly;
- Cloudflare R2 / S3-compatible storage is already used for release artifacts,
visual-regression assets, landing-page static assets, and mock recordings, so
the repository has S3-compatible operational precedent outside the product
runtime.
The gap is therefore not "does Open Design know how to talk to object storage?"
The gap is productizing that substrate for user attachments, generated
artifacts, access control, manifest indexing, retention, signed inspection URLs,
and Langfuse-safe observability payloads.
## Target Model
Open Design should treat observability as a closed loop:
```text
production trace
-> semantic task observations
-> automatic scores
-> agent triage and clustering
-> dataset or annotation candidate
-> experiment and regression gate
-> optimization proposal
-> human approval when risk is high
-> release
-> online observation
```
Langfuse is the trace, score, dataset, experiment, and annotation surface. It is
the observability index, not the durable storage layer for user attachments or
generated artifacts. PostHog remains the aggregate product analytics and
alerting surface. Open Design owns the domain model that maps a design-agent run
onto task stages, quality signals, storage references, and release decisions.
### Semantic Trace Shape
The target trace tree should expose product stages in addition to low-level
runtime spans:
```text
open-design-turn
brief-intake
attachment-manifest
route-task-kind
resolve-context
resolve-skill
resolve-design-system
resolve-memory
resolve-plugin
build-prompt-stack
spawn-agent
agent-work
plan
generate-artifact
edit-artifact
tool-call
verify-artifact
preview-render
artifact-manifest
export-check
critique
repair
evaluator
finalize
```
The exact implementation can preserve the existing `agent-run`, `llm`, and
`tool:*` observations, but those observations should sit inside or alongside
semantic task stages so humans and agents can diagnose by product intent.
### Heavy Input and Artifact Boundary
Open Design is a design product, so the input and output surface is heavier than
a short chat prompt. A run can include long briefs, reference images, PDFs,
brand files, prior project files, generated HTML, images, decks, documents,
preview screenshots, export bundles, and reproducibility materials. These
objects should not be copied wholesale into Langfuse input/output fields.
The target boundary is:
| Layer | Responsibility |
| --- | --- |
| Open Design object storage | Stores original user attachments, parsed text, thumbnails, OCR or embedding outputs, generated artifacts, preview screenshots, export bundles, and reproducibility files. |
| Langfuse | Stores trace-safe manifests, summaries, hashes, stage status, storage references, evaluator scores, usage, latency, and cost fields. |
| PostHog | Stores aggregate product analytics and user/workspace-level trend metrics derived from trace-safe fields. |
Langfuse should retain enough information for analysis, triage, and replay
without becoming the source of truth for sensitive or large user data. The
default policy is:
- do not upload original attachments or artifact packages to Langfuse by
default;
- always emit an `attachment_manifest` and `artifact_manifest` with stable ids,
MIME type, byte size, SHA-256 hash, parse/build status, truncation status,
redaction status, and Open Design storage references;
- include short summaries and product-analysis fields that are safe to index in
Langfuse;
- use short-lived signed URLs only for interactive inspection, not as the
durable reference stored in Langfuse;
- store durable `od://`-style references, object ids, or artifact ids that Open
Design can resolve after authorization;
- only upload a raw attachment to Langfuse Media when the file type is supported,
the data is low sensitivity or explicitly allowed, and replaying the trace
requires Langfuse-native media resolution;
- run masking/redaction before any input, output, or metadata leaves Open
Design.
The minimum manifest shape should be stable across traces:
```json
{
"attachment_manifest": [
{
"attachment_id": "att_123",
"role": "reference_image",
"mime_type": "image/png",
"size_bytes": 2481032,
"sha256": "sha256:...",
"parse_status": "ok",
"summary": "Brand reference image with dark UI styling.",
"redacted": false,
"truncated": false,
"stored_in_open_design": true,
"langfuse_media_id": null,
"storage_ref": "od://objects/workspaces/ws_1/projects/proj_1/runs/run_1/attachment/att_123"
}
],
"artifact_manifest": [
{
"artifact_id": "art_456",
"type": "html_prototype",
"mime_type": "text/html",
"size_bytes": 482193,
"sha256": "sha256:...",
"build_status": "ok",
"preview_status": "ok",
"export_status": "ok",
"summary": "Generated HTML prototype for the requested dashboard flow.",
"redacted": false,
"truncated": false,
"storage_ref": "od://objects/workspaces/ws_1/projects/proj_1/runs/run_1/artifact/art_456",
"thumbnail_ref": "od://objects/workspaces/ws_1/projects/proj_1/runs/run_1/preview/prev_thumb_789"
}
]
}
```
This boundary resolves the conflict between Langfuse truncation and incomplete
product analytics: Langfuse gets complete trace-safe semantics, while Open
Design keeps complete raw data behind product authorization and retention
policies.
### Attachment and Artifact Registry
Open Design should add a registry layer above `ProjectStorage` instead of
letting traces, filesystem paths, and UI URLs each invent their own storage
semantics. The registry is the product source of truth for durable input and
output objects; Langfuse stores only registry-derived manifest fields.
Recommended object classes:
| Object class | Examples | Registry owner |
| --- | --- | --- |
| `attachment` | Uploaded images, PDFs, brand files, reference assets, pasted files. | User/project input pipeline. |
| `parsed_attachment` | OCR text, extracted PDF text, thumbnails, embeddings, image descriptors. | Attachment ingest worker. |
| `artifact` | HTML prototypes, images, decks, documents, audio/video, code packages. | Agent/tool output pipeline. |
| `preview` | Rendered preview HTML, screenshots, thumbnails, validation captures. | Preview/verification pipeline. |
| `export` | Downloadable bundles, handoff packages, deployment artifacts. | Export/finalize pipeline. |
| `replay` | Minimal reproduction bundle, prompt stack snapshot, selected logs. | Observability/evaluation pipeline. |
Each registry record should have a stable id, object class, workspace/project/run
scope, storage backend, storage key, content hash, MIME type, byte size, origin,
retention policy, sensitivity class, redaction state, parse/build status,
preview/export status when applicable, and provenance back to the source trace
or run.
The storage reference in Langfuse should be durable and non-secret:
```text
od://objects/workspaces/<workspaceId>/projects/<projectId>/runs/<runId>/<objectClass>/<objectId>
```
The daemon should resolve that reference through product authorization and then,
only for interactive viewing or replay, issue a short-lived signed URL or stream
the object through a guarded daemon endpoint. Long-lived signed URLs should not
be written to Langfuse, PostHog, datasets, or PR comments.
The registry should use the existing `ProjectStorage` contract as the first
storage backend boundary, but the product routes need one more layer:
```text
HTTP / CLI / tool route
-> AttachmentRegistry or ArtifactRegistry
-> ProjectStorage backend (local by default, s3 when configured)
-> object manifest event
-> Langfuse trace-safe metadata
```
This keeps local-first desktop usage intact while making cloud-backed storage an
environment choice rather than a product model fork.
### Score Model
A score-eligible run is a terminal production agent run or accepted-dataset
experiment/evaluation run with enough trace metadata to identify trace id,
dataset item id when applicable, task kind, agent, model, final status, and
timing/token baselines. Local smoke tests and harness-only runs are excluded
unless they are recorded as accepted-dataset evaluation runs. Each eligible run
should receive automatic scores where the signal can be computed safely:
| Score | Meaning |
| --- | --- |
| `task_success` | The run produced a useful terminal result for the requested task. |
| `artifact_valid` | Required artifact files and manifest entries exist and are readable. |
| `preview_ok` | The primary preview renders without fatal runtime errors. |
| `user_request_covered` | The output appears to address the explicit user request. |
| `design_quality` | Automated or rubric-based critique result for visual/product quality. |
| `stability_risk` | Failure, retry, timeout, or brittle-runtime risk bucket. |
| `cost_bucket` | Cost/token use relative to task kind and model baseline. |
| `latency_bucket` | Latency relative to task kind and agent/model baseline. |
| `user_rating` | Human end-user feedback, already reported from the UI. |
Score applicability must be explicit so dashboards and datasets do not confuse
not-applicable scores with failures or evaluator gaps. When a score does not
apply, Open Design should not write a score value; it should record
`score_applicability.<score> = "not_applicable"`. When a score applies but cannot
be computed because required trace fields or evaluator inputs are missing, it
should record `score_applicability.<score> = "insufficient_signal"` and treat the
missing score as an observability gap, not as a failed run.
Accepted-dataset experiment and evaluation runs must reuse the same score fields,
score applicability values, and latency/cost bucket definitions as production
runs. Fixed datasets are the release-gate baseline; other accepted datasets use
the same fields for candidate improvement analysis instead of a separate QA-only
scoring schema.
| Score | Applies to |
| --- | --- |
| `task_success` | All score-eligible production and accepted-dataset experiment/evaluation runs with a terminal result. |
| `artifact_valid` | Artifact-producing score-eligible production and accepted-dataset experiment/evaluation tasks that create, edit, export, or inspect project files with a required manifest or file contract. |
| `preview_ok` | Artifact-producing score-eligible production and accepted-dataset experiment/evaluation tasks whose primary output has a supported preview renderer. |
| `user_request_covered` | Production and accepted-dataset experiment/evaluation tasks with a natural-language request or expected outcome and terminal output to compare. |
| `design_quality` | Score-eligible production and accepted-dataset experiment/evaluation outputs for visual, product, deck, prototype, image, video, audio, or document tasks with an applicable rubric. |
| `stability_risk` | All eligible runs. |
| `cost_bucket` | All score-eligible runs with model and token metadata. |
| `latency_bucket` | All score-eligible runs with timing metadata. |
| `user_rating` | Runs where the user submitted explicit feedback. |
Automatic scores should start conservative. Deterministic evaluators are
preferred before LLM-as-judge. Human feedback and annotation can later calibrate
or replace weak automatic scores.
### Dataset and Experiment Model
Production traces should become evaluation assets through an explicit promotion
path:
- failed traces with clear root cause become regression dataset candidates;
- low-score traces become quality dataset candidates;
- high-cost or high-latency traces become efficiency dataset candidates;
- ambiguous traces enter annotation queues before promotion;
- accepted dataset items retain source trace id, task kind, agent, model, skill,
design system, prompt stack fingerprint, trace-safe attachment/artifact
manifests, and expected outcome.
Experiments should run the same dataset against candidate changes such as:
- prompt stack edits;
- skill or design-system changes;
- agent/model routing changes;
- retry and runtime policy changes;
- context compression or cacheability changes;
- evaluator or guardrail changes.
An experiment is successful only when quality, reliability, latency, and cost
are considered together. A lower-cost run that fails the task is not an
improvement.
### Fixed Dataset Trust Contract
Fixed datasets are the trust anchor for human-agent collaboration. Agents can
discover issues, propose prompt/skill/model/runtime changes, and run the
experiments, but humans should not have to accept those proposals on narrative
confidence alone. A fixed dataset gives the team a shared, repeatable evidence
surface before a change reaches production.
For every optimization proposal, the agent should report:
- which fixed dataset or approved provisional baseline was used;
- which candidate changed, including git sha, agent/model routing, prompt stack
fingerprint, skill/design-system versions, and evaluator version;
- pass/fail status for each blocking quality, reliability, latency, and cost
metric;
- the concrete traces, dataset items, artifacts, and annotations behind any
failure or tradeoff;
- whether any sensitive attachment or generated artifact entered the evaluation
fixture set, and who approved that promotion.
The default trust rule is conservative:
- quality must not regress on fixed task datasets;
- stability must not regress;
- latency and cost must improve or stay flat, unless a human explicitly accepts
a quality tradeoff;
- sensitive artifact or attachment cases cannot become long-lived fixtures
without human approval.
This contract turns observability into a collaboration mechanism. The agent does
the repetitive measurement work; the human reviews evidence, calibrates
judgment-heavy evaluators, and approves only the risk that cannot be reduced to
data.
## Human and Agent Responsibilities
The loop is AI Native because agents should operate the routine diagnosis and
validation work, while humans intervene at high-leverage judgment points.
Agent-owned work:
- scan recent traces and dashboards on a schedule;
- rank failures by category, detail, stage, agent, model, OS, and task kind;
- cluster low-score or high-cost traces into failure modes;
- propose dataset and annotation candidates;
- run experiments for candidate prompt, skill, model, or runtime changes;
- summarize pass/fail evidence and tradeoffs across quality, reliability,
latency, and cost;
- draft optimization PRs or prompt/skill changes when the risk is low.
Human-owned work:
- approve production releases and high-risk behavior changes;
- approve provisional baselines and sensitive fixture promotion;
- label ambiguous or subjective quality cases;
- approve major system prompt, skill, or design-system changes;
- decide quality versus cost tradeoffs when metrics conflict;
- review annotation queues and calibrate evaluators.
The intended working loop is:
```text
agent finds a signal
-> agent explains the likely cause
-> agent proposes data/evaluator/prompt/runtime changes
-> fixed-dataset experiment proves or rejects the proposal
-> human reviews the pass/fail evidence and approves only the meaningful risk
```
## Metrics
The loop should report progress through a small set of durable metrics.
Quality:
- `task_success` by task kind, agent, model, skill, and design system;
- `preview_ok` and `artifact_valid` for artifact-producing tasks;
- user rating rate and negative-reason distribution;
- annotation agreement or evaluator drift where available.
Reliability:
- terminal failure rate;
- unknown or unattributable failure share;
- retryable failure share and retry success rate;
- failure category/detail/stage distribution.
Latency:
- P50/P90/P99 total duration by task kind;
- slowest segment by queue, prompt build, spawn, first token, generation, tool
aggregate, verify, and finalize;
- preview and artifact verification duration once those stages are observable.
Cost:
- provider input tokens, effective input tokens, output tokens, and total tokens;
- estimated context tokens and uncached input tokens;
- cache hit ratio and cache source;
- calculated cost when Langfuse pricing supports the model;
- fallback cost bucket when a model has no pricing mapping.
Loop health:
- dataset item count by source and task kind;
- experiment count and pass/fail trend;
- annotation queue throughput and aging;
- number of shipped changes linked to observed trace evidence.
Data completeness:
- share of artifact-producing runs with a valid `artifact_manifest`;
- share of attachment-bearing runs with a valid `attachment_manifest`;
- share of traces with truncated input/output and no compensating summary;
- share of manifests with missing storage references, hashes, or parse/build
status;
- masked or redacted attachment share by task kind and data class.
## Rollout Slices
### Slice 1: Spec and Baseline
- Land this spec.
- Keep runtime behavior unchanged.
- Document current trace coverage, score coverage, dataset count, and known gaps
from Langfuse/PostHog.
### Slice 2: Semantic Stage Observations
- Add task-stage observations around existing run lifecycle boundaries.
- Preserve existing trace ids, tags, and low-level observations.
- Emit stage status, duration, and failure metadata so failed traces can be
grouped by product stage.
- Emit trace-safe attachment and artifact manifests during `brief-intake` and
`verify-artifact`.
### Slice 3: Object Storage and Manifest Contract
- Define the Open Design object storage contract for attachments, parsed
derivatives, generated artifacts, preview screenshots, export bundles, and
reproducibility files.
- Define the registry schema and `od://` reference format for `attachment`,
`parsed_attachment`, `artifact`, `preview`, `export`, and `replay` objects.
- Store original heavy inputs and outputs in Open Design controlled storage,
not in Langfuse by default.
- Store durable object references, hashes, summaries, redaction state,
truncation state, and parse/build status in Langfuse.
- Wire user upload, project files, standalone artifact save, live artifact,
media generation, preview verification, and export/finalize flows through the
registry instead of writing unindexed files only.
- Reuse `ProjectStorage` as the backend adapter boundary: local remains the
default, while S3-compatible storage is enabled by environment configuration
once product authorization, retention, and signed inspection URLs are in
place.
- Add masking before Langfuse ingestion so raw sensitive content cannot leak
through input, output, or metadata fields.
### Slice 4: Automatic Evaluators
- Add deterministic scores for artifact validity, preview success, task success
proxy, latency bucket, and cost bucket.
- Keep evaluator failures non-blocking.
- Write scores back to Langfuse and mirror aggregate fields to PostHog where
dashboarding needs them.
### Slice 5: Dataset and Annotation Promotion
- Add an agent-operable workflow that proposes dataset items from failed,
low-score, high-cost, and high-latency traces.
- Route ambiguous quality cases to Langfuse annotation queues.
- Preserve provenance from dataset item back to source trace and accepted human
annotation.
- Preserve trace-safe manifest references for attachments and artifacts; require
human approval before promoting sensitive raw attachments or artifacts into
durable evaluation fixtures.
### Slice 6: Experiment and Release Gates
- Define fixed task datasets for core task kinds.
- Run experiments before prompt, skill, model-routing, retry, or context changes
are treated as durable wins.
- Gate releases on quality not regressing while reliability, latency, or cost
improves.
- Treat fixed-dataset pass/fail evidence as the default trust artifact for
product and engineering review.
The release gate should use a fixed comparator so the same experiment result
produces the same pass/fail decision across implementations:
| Gate field | Rule |
| --- | --- |
| Baseline comparator | Compare the candidate against the latest approved release on the fixed dataset for each affected task kind. If no fixed dataset exists yet, compare against the last 14 days of production traces promoted into the baseline window and block shipping until a maintainer approves that provisional baseline. |
| Blocking quality metrics | `task_success`, `user_request_covered`, and applicable `artifact_valid` / `preview_ok` / `design_quality`. Any blocking quality metric regressing by more than 1 percentage point, or producing one additional critical artifact failure in a fixed regression dataset, blocks the release. |
| Blocking reliability metrics | Terminal failure rate, unknown failure share, retryable failure share, and retry success rate. A candidate fails if terminal or unknown failures increase by more than 1 percentage point, or if retry success drops by more than 2 percentage points. |
| Blocking latency and cost metrics | P90 total duration and total token or calculated cost by task kind. A candidate fails if either P90 latency or cost increases by more than 5% without an explicit human-approved quality tradeoff. |
| Advisory metrics | P50 latency, P99 latency, cache hit ratio, annotation throughput, dataset growth, and user rating volume. These must be reported with the gate result but do not block by themselves. |
| Improvement threshold | A candidate qualifies as an improvement only when all blocking metrics pass and at least one blocking reliability, latency, or cost metric improves by 5% or more, or one documented failure category is eliminated on the fixed regression dataset. |
### Slice 7: Agent-Operated Optimization Loop
- Schedule an agent report that ranks the most important quality, reliability,
latency, and cost opportunities.
- Let agents draft low-risk improvement PRs with linked trace, dataset, and
experiment evidence.
- Require human approval for high-risk prompts, guardrails, model-routing, and
product-quality decisions.
## Relationship to Existing Plans
`specs/current/run-reliability-optimization-plan.md` remains the concrete plan
for reducing failures and optimizing latency/token cost after the first
observability slice. This spec is the higher-level loop that makes that work
repeatable and connects it to quality evaluation, datasets, experiments,
annotation, and human-agent collaboration.
`specs/current/automation-self-evolution.md` describes how successful runs and
sources can promote durable memory, skills, design systems, and automation
templates. This observability loop provides the evidence and gates for deciding
which proposed evolutions are trustworthy.
## Non-goals
- This spec does not replace Langfuse or PostHog.
- This spec does not make Langfuse the storage layer for original user
attachments or generated artifact packages.
- This spec does not add a new model router.
- This spec does not require all evaluators to be LLM judges.
- This spec does not implement runtime changes in the spec-only PR.
- This spec does not make agents auto-ship high-risk changes without human
approval.
## Acceptance Criteria for Follow-up Implementation
The loop is working when a representative production regression can be handled
end to end:
1. A bad run produces a trace with a clear failing task stage.
2. Attachment and artifact manifests include trace-safe summaries, hashes,
storage references, truncation state, and redaction state when applicable.
3. Automatic or human scores identify the quality, stability, latency, or cost
problem.
4. The trace is promoted to a dataset item or annotation queue with provenance.
5. An experiment compares a proposed fix against the accepted dataset.
6. The result summarizes quality, reliability, latency, cost, and data
completeness tradeoffs.
7. A human approves high-risk changes or the agent proceeds with a low-risk
improvement.
8. Online dashboards confirm the improvement after release.