Files
open-design/apps/web/tests/analytics-configure-globals.test.ts
680cc818bb feat(analytics): runtime_type across the full event chain + BYOK run events (#4510)
* feat(analytics): runtime_type across the full event chain + BYOK run events

Make runtime_type (amr_cloud / byok / local_cli) a first-class analytics
dimension on every event so the behavioural funnel can split AMR / BYOK / CLI
without static cohorts.

- contracts: add runtime_type to AnalyticsConfigureGlobals (derived in
  deriveConfigureGlobals as the single active runtime — no 'both' cascade) and
  ChatAnalyticsHints; widen run agent_provider_id to allow BYOK providers.
- daemon: stamp runtime_type on run_created/run_finished, with the web client's
  per-run hint overriding the BYOK-blind server derivation
  (runtimeTypeForRunAnalytics).
- web: register runtime_type as a super-property so all client events carry it;
  emit run_created/run_finished for BYOK runs, which stream client-side and so
  never produced daemon run events before (they were invisible to the funnel).

Red-spec first: apps/daemon/tests/run-runtime-type-analytics.test.ts pins the
hint-override; apps/web/tests/byok-run-analytics.test.ts pins the BYOK builders.

* fix(analytics): address review — pin BYOK runtime_type, guard run_finished, mode-first derivation

- byok-run: stamp runtime_type='byok' on run_created/run_finished instead of
  relying on the mutable super-property, so a mid-stream mode switch can't
  split one run across runtime buckets. Adds optional runtime_type to
  RunCreatedProps for the per-event override (daemon already pins it).
- ProjectView: wrap the artifact-count refresh in try/catch so a rejected
  refreshProjectFiles() no longer swallows the success run_finished.
- deriveConfigureGlobals: check mode==='api' before agentId==='amr' so an
  AMR→BYOK switch (which leaves agentId='amr') reports runtime_type=byok, not
  amr_cloud. Adds a regression test.

* ci: re-trigger after cooldown

* ci: re-trigger after cancelling stuck run

---------

Co-authored-by: free666799 <293857035+free666799@users.noreply.github.com>
2026-06-19 15:01:22 +00:00

377 lines
12 KiB
TypeScript

// Regression test for the v2 configure-state globals
// (has_available_configure_cli / configure_type / configure_availability).
//
// Reviewer comment on PR #2285 (mrcfps, 2026-05-19) flagged that
// `setConfigureGlobals` was defined but never called, so every browser
// capture inherited the boot defaults `{ false, 'unknown', 'unknown' }`.
// App.tsx now drives the setter from a useEffect that watches mode /
// agentId / apiKey / apiProtocolConfigs / agents; these tests pin the
// derive-then-register behavior end-to-end against the client module so a
// future refactor can't silently regress it back to a no-op setter.
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import {
deriveConfigureGlobals,
type DeriveConfigureGlobalsInput,
} from '@open-design/contracts/analytics';
import {
getConfigureGlobals,
setConfigureGlobals,
} from '../src/analytics/client';
const BOOT_DEFAULTS = {
has_available_configure_cli: false,
configure_type: 'unknown' as const,
configure_availability: 'unknown' as const,
runtime_type: 'none' as const,
cli_runnable: false,
byok_runnable: false,
amr_runnable: false,
};
describe('deriveConfigureGlobals', () => {
it('returns "none" / "unknown" when nothing is configured', () => {
expect(deriveConfigureGlobals({})).toEqual({
has_available_configure_cli: false,
configure_type: 'none',
configure_availability: 'unknown',
runtime_type: 'none',
cli_runnable: false,
byok_runnable: false,
amr_runnable: false,
});
});
it('reports local_cli when an installed CLI is the selected agent in daemon mode', () => {
const input: DeriveConfigureGlobalsInput = {
mode: 'daemon',
agentId: 'claude',
agents: [
{ id: 'claude', available: true },
{ id: 'codex', available: false },
],
};
expect(deriveConfigureGlobals(input)).toEqual({
has_available_configure_cli: true,
configure_type: 'local_cli',
configure_availability: 'available',
runtime_type: 'local_cli',
cli_runnable: true,
byok_runnable: false,
amr_runnable: false,
});
});
it('marks the configure as unavailable when the selected daemon-mode agent is not installed', () => {
const input: DeriveConfigureGlobalsInput = {
mode: 'daemon',
agentId: 'codex',
agents: [
{ id: 'claude', available: true },
{ id: 'codex', available: false },
],
};
expect(deriveConfigureGlobals(input)).toMatchObject({
configure_type: 'local_cli',
configure_availability: 'unavailable',
});
});
it('runtime_type follows api mode over a stale remembered amr agentId', () => {
// Switching AMR → BYOK only flips config.mode; config.agentId can stay
// 'amr'. The active runtime is BYOK, so runtime_type must be 'byok', not
// 'amr_cloud'. Regression for the AMR→Use-API funnel mislabel.
expect(
deriveConfigureGlobals({ mode: 'api', agentId: 'amr', byokConfigured: true }).runtime_type,
).toBe('byok');
});
it('reports byok when an api-mode user has saved credentials', () => {
expect(
deriveConfigureGlobals({
mode: 'api',
byokConfigured: true,
agents: [],
}),
).toEqual({
has_available_configure_cli: false,
configure_type: 'byok',
configure_availability: 'available',
runtime_type: 'byok',
cli_runnable: false,
byok_runnable: true,
amr_runnable: false,
});
});
it('reports both when api-mode user also has CLIs installed', () => {
expect(
deriveConfigureGlobals({
mode: 'api',
byokConfigured: true,
agents: [{ id: 'claude', available: true }],
}),
).toMatchObject({
has_available_configure_cli: true,
configure_type: 'both',
});
});
});
describe('deriveConfigureGlobals — cold-start gating', () => {
// Reviewer #2285 (mrcfps, 2026-05-20 03:10) flagged a cold-start hole:
// `fetchAgentsStream()` is async, so the App-level useEffect first fires with
// `agents=[]` and `mode='daemon'`. Without gating, the helper used to
// stamp `has_available_configure_cli=false / configure_availability=
// unavailable` on every page_view fired before the probe resolved.
// App.tsx now waits on `agentsLoading === false` and leaves the boot
// defaults in place; these tests pin what the helper SHOULD return
// for the empty-agents / partial-config inputs so a future caller
// can't silently skip the gate again.
it('reports unavailable / none when daemon mode is pinned but no agents are loaded yet', () => {
// Pre-amr_auth_result, the daemon branch hardcoded 'local_cli', which
// made 'none' unreachable on desktop. The type now follows the actual
// configured state; App.tsx's agentsLoading gate still keeps the boot
// 'unknown' in place until the probe lands, so this input shape only
// reaches the helper for machines that genuinely have no CLI.
expect(
deriveConfigureGlobals({ mode: 'daemon', agentId: 'claude', agents: [] }),
).toEqual({
has_available_configure_cli: false,
configure_type: 'none',
configure_availability: 'unavailable',
runtime_type: 'none',
cli_runnable: false,
byok_runnable: false,
amr_runnable: false,
});
});
it('reports has_available_configure_cli=true once the probe lands and the selected agent is installed', () => {
expect(
deriveConfigureGlobals({
mode: 'daemon',
agentId: 'claude',
agents: [{ id: 'claude', available: true }],
}),
).toEqual({
has_available_configure_cli: true,
configure_type: 'local_cli',
configure_availability: 'available',
runtime_type: 'local_cli',
cli_runnable: true,
byok_runnable: false,
amr_runnable: false,
});
});
it('survives an undefined agents list (caller forgot to pass)', () => {
expect(
deriveConfigureGlobals({ mode: 'daemon', agentId: 'claude' }),
).toMatchObject({
has_available_configure_cli: false,
configure_availability: 'unavailable',
});
});
// Reviewer #2285 (mrcfps, 2026-05-20 03:36) flagged the daemon call site
// for not pinning `mode: 'daemon'`. Without the explicit mode the helper
// falls through to the generic branch and reports `available` whenever
// any unrelated CLI is on PATH — even when the requested agent is the
// one that's missing. This test pins the right behavior: a run targeted
// at an uninstalled agent must report `unavailable`, regardless of
// whether other agents happen to be installed.
it('reports unavailable when the requested daemon-mode agent is missing even if peers are installed', () => {
expect(
deriveConfigureGlobals({
mode: 'daemon',
agentId: 'codex',
agents: [
{ id: 'claude', available: true },
{ id: 'codex', available: false },
],
}),
).toEqual({
has_available_configure_cli: true,
configure_type: 'local_cli',
configure_availability: 'unavailable',
runtime_type: 'local_cli',
cli_runnable: true,
byok_runnable: false,
amr_runnable: false,
});
});
});
describe('deriveConfigureGlobals — AMR', () => {
// AMR ships with the app, so its agent row counting as a "local CLI"
// would put every install in the local_cli bucket and make the
// amr/none buckets unreachable. Sign-in (`amrAuthorized`) is AMR's
// configured signal.
it('does not count the bundled amr agent as an available local CLI', () => {
expect(
deriveConfigureGlobals({
mode: 'daemon',
agentId: 'amr',
agents: [{ id: 'amr', available: true }],
}),
).toEqual({
has_available_configure_cli: false,
configure_type: 'none',
configure_availability: 'available',
runtime_type: 'amr_cloud',
cli_runnable: false,
byok_runnable: false,
amr_runnable: false,
});
});
it('reports amr when sign-in is the only configured path', () => {
expect(
deriveConfigureGlobals({
mode: 'daemon',
agentId: 'amr',
agents: [{ id: 'amr', available: true }],
amrAuthorized: true,
}),
).toEqual({
has_available_configure_cli: false,
configure_type: 'amr',
configure_availability: 'available',
runtime_type: 'amr_cloud',
cli_runnable: false,
byok_runnable: false,
amr_runnable: true,
});
});
it('keeps local_cli precedence when a real CLI is installed alongside AMR auth', () => {
expect(
deriveConfigureGlobals({
mode: 'daemon',
agentId: 'claude',
agents: [
{ id: 'claude', available: true },
{ id: 'amr', available: true },
],
amrAuthorized: true,
}),
).toMatchObject({
has_available_configure_cli: true,
configure_type: 'local_cli',
});
});
it('reports amr / available with no mode pinned when only AMR is authorized', () => {
expect(
deriveConfigureGlobals({ amrAuthorized: true })).toEqual({
has_available_configure_cli: false,
configure_type: 'amr',
configure_availability: 'available',
runtime_type: 'amr_cloud',
cli_runnable: false,
byok_runnable: false,
amr_runnable: true,
});
});
});
describe('deriveConfigureGlobals — independent runnable flags', () => {
// The whole point of the runnable trio: `configure_type` is a priority
// cascade that masks lower-priority paths (CLI + BYOK collapses to 'both',
// and AMR never shows when a CLI/BYOK exists), so per-path activation can't
// be read off it. `cli_runnable` / `byok_runnable` / `amr_runnable` are
// independent, so a fully-configured user lights up all three even while
// `configure_type` reports the single cascade winner.
it('reports all three runnable flags independently when CLI + BYOK + AMR are all configured', () => {
expect(
deriveConfigureGlobals({
mode: 'daemon',
agentId: 'claude',
agents: [
{ id: 'claude', available: true },
{ id: 'amr', available: true },
],
byokConfigured: true,
amrAuthorized: true,
}),
).toEqual({
// cascade winner masks BYOK and AMR …
has_available_configure_cli: true,
configure_type: 'both',
configure_availability: 'available',
runtime_type: 'local_cli',
// … but the independent flags do not.
cli_runnable: true,
byok_runnable: true,
amr_runnable: true,
});
});
it('lights up amr_runnable even though configure_type hides AMR behind a CLI', () => {
expect(
deriveConfigureGlobals({
mode: 'daemon',
agentId: 'claude',
agents: [
{ id: 'claude', available: true },
{ id: 'amr', available: true },
],
amrAuthorized: true,
}),
).toMatchObject({
configure_type: 'local_cli',
cli_runnable: true,
byok_runnable: false,
amr_runnable: true,
});
});
});
describe('setConfigureGlobals (web client)', () => {
// Reset the module-level state so other suites do not bleed in.
beforeEach(() => {
setConfigureGlobals(BOOT_DEFAULTS);
});
afterEach(() => {
setConfigureGlobals(BOOT_DEFAULTS);
});
it('stores the latest configure-state for downstream captures', () => {
expect(getConfigureGlobals()).toEqual(BOOT_DEFAULTS);
setConfigureGlobals({
has_available_configure_cli: true,
configure_type: 'local_cli',
configure_availability: 'available',
runtime_type: 'local_cli',
cli_runnable: true,
byok_runnable: false,
amr_runnable: false,
});
expect(getConfigureGlobals()).toEqual({
has_available_configure_cli: true,
configure_type: 'local_cli',
configure_availability: 'available',
runtime_type: 'local_cli',
cli_runnable: true,
byok_runnable: false,
amr_runnable: false,
});
});
it('never throws when no PostHog client is initialized', () => {
expect(() =>
setConfigureGlobals({
has_available_configure_cli: true,
configure_type: 'both',
configure_availability: 'available',
runtime_type: 'local_cli',
cli_runnable: true,
byok_runnable: true,
amr_runnable: false,
}),
).not.toThrow();
});
});