test: comprehensive analytics test suite — unit, API, E2E, air-gapped

tests/e2e-docker/analytics-disabled.spec.ts

@@ -7,7 +7,7 @@ import { expect, test } from "@playwright/test";
 // If the container has analytics enabled, these tests will be skipped
 // automatically by checking the config endpoint first.
 
-const BASE_URL = "http://localhost:1349";
+const BASE_URL = process.env.API_URL ?? "http://localhost:1349";
 
 async function loginFresh(page: import("@playwright/test").Page) {
   await page.goto("/login");

tests/e2e-docker/auth.setup.ts

@@ -1,12 +1,25 @@
+import path from "node:path";
 import { expect, test as setup } from "@playwright/test";
-import { authFile } from "../../playwright.docker.config";
+
+const authFile =
+  // Support both playwright.docker.config and playwright.analytics.config
+  path.join(__dirname, "..", "..", "test-results", ".auth", "analytics-user.json");
 
 setup("authenticate", async ({ page }) => {
   await page.goto("/login");
   await page.getByLabel("Username").fill("admin");
   await page.getByLabel("Password").fill("admin");
   await page.getByRole("button", { name: /login/i }).click();
-  await page.waitForURL("/", { timeout: 30_000 });
+
+  // After login, may land on "/" or "/analytics-consent" (fresh user)
+  await page.waitForURL(/\/(analytics-consent)?$/, { timeout: 30_000 });
+
+  // If redirected to consent page, accept analytics to proceed
+  if (page.url().includes("/analytics-consent")) {
+    await page.getByRole("button", { name: /sure, sounds good/i }).click();
+    await page.waitForURL("/", { timeout: 15_000 });
+  }
+
   await expect(page).toHaveURL("/");
   await page.context().storageState({ path: authFile });
 });

tests/unit/analytics-consent-extended.test.ts

@@ -0,0 +1,183 @@
+import type { ConsentState } from "@ashim/shared";
+import { isConsentEnabled, shouldShowConsent } from "@ashim/shared";
+import { describe, expect, it } from "vitest";
+
+const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000;
+
+describe("shouldShowConsent edge cases", () => {
+  it("returns true when remindAt is exactly equal to Date.now()", () => {
+    const now = Date.now();
+    const state: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: now - SEVEN_DAYS_MS,
+      analyticsConsentRemindAt: now,
+    };
+    // Date.now() >= remindAt should be true when they are equal
+    expect(shouldShowConsent(state, true)).toBe(true);
+  });
+
+  it("returns true when remindAt is set but consentShownAt is null (defensive)", () => {
+    const state: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: null,
+      analyticsConsentRemindAt: Date.now() - 1000,
+    };
+    // consentShownAt is null -> returns true (fresh user path)
+    expect(shouldShowConsent(state, true)).toBe(true);
+  });
+
+  it("returns false when remindAt is 1ms in the future", () => {
+    const now = Date.now();
+    const state: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: now - SEVEN_DAYS_MS,
+      analyticsConsentRemindAt: now + 100000, // safely in the future
+    };
+    expect(shouldShowConsent(state, true)).toBe(false);
+  });
+
+  it("returns true after 'Maybe later' and 7 days have passed", () => {
+    const shownAt = Date.now() - SEVEN_DAYS_MS - 1000;
+    const remindAt = Date.now() - 1000; // remind time has passed
+    const state: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: shownAt,
+      analyticsConsentRemindAt: remindAt,
+    };
+    expect(shouldShowConsent(state, true)).toBe(true);
+  });
+
+  it("returns false when server is disabled regardless of remindAt", () => {
+    const state: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: Date.now() - SEVEN_DAYS_MS,
+      analyticsConsentRemindAt: Date.now() - 1000,
+    };
+    expect(shouldShowConsent(state, false)).toBe(false);
+  });
+});
+
+describe("isConsentEnabled edge cases", () => {
+  it("returns false when analyticsEnabled is false (explicitly declined)", () => {
+    const state: ConsentState = {
+      analyticsEnabled: false,
+      analyticsConsentShownAt: Date.now(),
+      analyticsConsentRemindAt: null,
+    };
+    expect(isConsentEnabled(state, true)).toBe(false);
+  });
+
+  it("returns false when analyticsEnabled is null (never decided)", () => {
+    const state: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: null,
+      analyticsConsentRemindAt: null,
+    };
+    expect(isConsentEnabled(state, true)).toBe(false);
+  });
+
+  it("returns false when server disabled even if user opted in", () => {
+    const state: ConsentState = {
+      analyticsEnabled: true,
+      analyticsConsentShownAt: Date.now(),
+      analyticsConsentRemindAt: null,
+    };
+    expect(isConsentEnabled(state, false)).toBe(false);
+  });
+});
+
+describe("consent lifecycle simulations", () => {
+  it("fresh -> maybe later -> remind time passes -> show again -> accept", () => {
+    // Step 1: Fresh user -- never been asked
+    const fresh: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: null,
+      analyticsConsentRemindAt: null,
+    };
+    expect(shouldShowConsent(fresh, true)).toBe(true);
+    expect(isConsentEnabled(fresh, true)).toBe(false);
+
+    // Step 2: User clicks "Maybe later" -- shown timestamp set, remind in 7 days
+    const shownAt = Date.now() - SEVEN_DAYS_MS - 1000;
+    const maybeLater: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: shownAt,
+      analyticsConsentRemindAt: shownAt + SEVEN_DAYS_MS,
+    };
+    // Remind time has now passed (shownAt + 7days < now)
+    expect(shouldShowConsent(maybeLater, true)).toBe(true);
+    expect(isConsentEnabled(maybeLater, true)).toBe(false);
+
+    // Step 3: User accepts on second prompt
+    const accepted: ConsentState = {
+      analyticsEnabled: true,
+      analyticsConsentShownAt: Date.now(),
+      analyticsConsentRemindAt: null,
+    };
+    expect(shouldShowConsent(accepted, true)).toBe(false);
+    expect(isConsentEnabled(accepted, true)).toBe(true);
+  });
+
+  it("fresh -> accept immediately -> never show again", () => {
+    // Step 1: Fresh user
+    const fresh: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: null,
+      analyticsConsentRemindAt: null,
+    };
+    expect(shouldShowConsent(fresh, true)).toBe(true);
+
+    // Step 2: User accepts immediately
+    const accepted: ConsentState = {
+      analyticsEnabled: true,
+      analyticsConsentShownAt: Date.now(),
+      analyticsConsentRemindAt: null,
+    };
+    expect(shouldShowConsent(accepted, true)).toBe(false);
+    expect(isConsentEnabled(accepted, true)).toBe(true);
+
+    // Verify it stays hidden even far in the future
+    expect(shouldShowConsent(accepted, true)).toBe(false);
+  });
+
+  it("fresh -> decline immediately -> never show again", () => {
+    // Step 1: Fresh user
+    const fresh: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: null,
+      analyticsConsentRemindAt: null,
+    };
+    expect(shouldShowConsent(fresh, true)).toBe(true);
+
+    // Step 2: User declines immediately
+    const declined: ConsentState = {
+      analyticsEnabled: false,
+      analyticsConsentShownAt: Date.now(),
+      analyticsConsentRemindAt: null,
+    };
+    expect(shouldShowConsent(declined, true)).toBe(false);
+    expect(isConsentEnabled(declined, true)).toBe(false);
+
+    // Verify it stays hidden and analytics stays disabled
+    expect(shouldShowConsent(declined, true)).toBe(false);
+    expect(isConsentEnabled(declined, true)).toBe(false);
+  });
+
+  it("fresh -> maybe later -> remind time NOT yet passed -> stay hidden", () => {
+    const fresh: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: null,
+      analyticsConsentRemindAt: null,
+    };
+    expect(shouldShowConsent(fresh, true)).toBe(true);
+
+    // User clicks maybe later, only 1 day ago
+    const maybeLater: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: Date.now() - 86400000,
+      analyticsConsentRemindAt: Date.now() + SEVEN_DAYS_MS - 86400000,
+    };
+    expect(shouldShowConsent(maybeLater, true)).toBe(false);
+    expect(isConsentEnabled(maybeLater, true)).toBe(false);
+  });
+});

tests/unit/analytics-events.test.ts

@@ -0,0 +1,61 @@
+import { ANALYTICS_EVENTS } from "@ashim/shared";
+import { describe, expect, it } from "vitest";
+
+describe("ANALYTICS_EVENTS", () => {
+  it("has exactly 4 event keys", () => {
+    expect(Object.keys(ANALYTICS_EVENTS)).toHaveLength(4);
+  });
+
+  it("contains the expected keys", () => {
+    expect(ANALYTICS_EVENTS).toHaveProperty("TOOL_USED");
+    expect(ANALYTICS_EVENTS).toHaveProperty("SEARCH");
+    expect(ANALYTICS_EVENTS).toHaveProperty("PIPELINE_EXECUTED");
+    expect(ANALYTICS_EVENTS).toHaveProperty("AI_BUNDLE_ACTION");
+  });
+
+  it("all event values are strings", () => {
+    for (const value of Object.values(ANALYTICS_EVENTS)) {
+      expect(typeof value).toBe("string");
+    }
+  });
+
+  it("TOOL_USED has the correct snake_case value", () => {
+    expect(ANALYTICS_EVENTS.TOOL_USED).toBe("tool_used");
+  });
+
+  it("SEARCH has the correct snake_case value", () => {
+    expect(ANALYTICS_EVENTS.SEARCH).toBe("search");
+  });
+
+  it("PIPELINE_EXECUTED has the correct snake_case value", () => {
+    expect(ANALYTICS_EVENTS.PIPELINE_EXECUTED).toBe("pipeline_executed");
+  });
+
+  it("AI_BUNDLE_ACTION has the correct snake_case value", () => {
+    expect(ANALYTICS_EVENTS.AI_BUNDLE_ACTION).toBe("ai_bundle_action");
+  });
+
+  it("all values follow snake_case convention", () => {
+    for (const value of Object.values(ANALYTICS_EVENTS)) {
+      expect(value).toMatch(/^[a-z][a-z0-9_]*$/);
+    }
+  });
+
+  it("is frozen (as const prevents mutation)", () => {
+    // as const produces a readonly object; Object.isFrozen checks runtime freezing.
+    // TypeScript enforces readonly at compile time, but at runtime the object
+    // defined with "as const" is a plain object unless explicitly frozen.
+    // We verify the values are stable by checking they haven't changed.
+    const snapshot = { ...ANALYTICS_EVENTS };
+    expect(ANALYTICS_EVENTS.TOOL_USED).toBe(snapshot.TOOL_USED);
+    expect(ANALYTICS_EVENTS.SEARCH).toBe(snapshot.SEARCH);
+    expect(ANALYTICS_EVENTS.PIPELINE_EXECUTED).toBe(snapshot.PIPELINE_EXECUTED);
+    expect(ANALYTICS_EVENTS.AI_BUNDLE_ACTION).toBe(snapshot.AI_BUNDLE_ACTION);
+  });
+
+  it("all values are unique (no duplicate event names)", () => {
+    const values = Object.values(ANALYTICS_EVENTS);
+    const unique = new Set(values);
+    expect(unique.size).toBe(values.length);
+  });
+});

tests/unit/analytics-types.test.ts

@@ -0,0 +1,129 @@
+import type { AnalyticsConfig, ConsentState } from "@ashim/shared";
+import { describe, expect, it } from "vitest";
+
+describe("AnalyticsConfig type", () => {
+  it("accepts a fully populated config object", () => {
+    const config: AnalyticsConfig = {
+      enabled: true,
+      posthogApiKey: "phc_test123",
+      posthogHost: "https://us.i.posthog.com",
+      sentryDsn: "https://abc@sentry.io/123",
+      sampleRate: 1.0,
+      instanceId: "inst-abc-123",
+    };
+    expect(config.enabled).toBe(true);
+    expect(config.posthogApiKey).toBe("phc_test123");
+    expect(config.posthogHost).toBe("https://us.i.posthog.com");
+    expect(config.sentryDsn).toBe("https://abc@sentry.io/123");
+    expect(config.sampleRate).toBe(1.0);
+    expect(config.instanceId).toBe("inst-abc-123");
+  });
+
+  it("accepts a config with analytics disabled", () => {
+    const config: AnalyticsConfig = {
+      enabled: false,
+      posthogApiKey: "",
+      posthogHost: "",
+      sentryDsn: "",
+      sampleRate: 0,
+      instanceId: "",
+    };
+    expect(config.enabled).toBe(false);
+    expect(config.sampleRate).toBe(0);
+  });
+
+  it("accepts fractional sample rates", () => {
+    const config: AnalyticsConfig = {
+      enabled: true,
+      posthogApiKey: "key",
+      posthogHost: "https://host.com",
+      sentryDsn: "https://dsn",
+      sampleRate: 0.5,
+      instanceId: "id",
+    };
+    expect(config.sampleRate).toBe(0.5);
+  });
+
+  it("has exactly the expected keys", () => {
+    const config: AnalyticsConfig = {
+      enabled: true,
+      posthogApiKey: "key",
+      posthogHost: "host",
+      sentryDsn: "dsn",
+      sampleRate: 1,
+      instanceId: "id",
+    };
+    const keys = Object.keys(config).sort();
+    expect(keys).toEqual(
+      ["enabled", "instanceId", "posthogApiKey", "posthogHost", "sampleRate", "sentryDsn"].sort(),
+    );
+  });
+});
+
+describe("ConsentState type", () => {
+  it("accepts all-null state (fresh user)", () => {
+    const state: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: null,
+      analyticsConsentRemindAt: null,
+    };
+    expect(state.analyticsEnabled).toBeNull();
+    expect(state.analyticsConsentShownAt).toBeNull();
+    expect(state.analyticsConsentRemindAt).toBeNull();
+  });
+
+  it("accepts opted-in state (analyticsEnabled = true)", () => {
+    const state: ConsentState = {
+      analyticsEnabled: true,
+      analyticsConsentShownAt: 1713800000000,
+      analyticsConsentRemindAt: null,
+    };
+    expect(state.analyticsEnabled).toBe(true);
+    expect(state.analyticsConsentShownAt).toBe(1713800000000);
+    expect(state.analyticsConsentRemindAt).toBeNull();
+  });
+
+  it("accepts declined state (analyticsEnabled = false)", () => {
+    const state: ConsentState = {
+      analyticsEnabled: false,
+      analyticsConsentShownAt: 1713800000000,
+      analyticsConsentRemindAt: null,
+    };
+    expect(state.analyticsEnabled).toBe(false);
+  });
+
+  it("accepts deferred state (maybe later with remindAt set)", () => {
+    const shownAt = Date.now() - 86400000;
+    const remindAt = Date.now() + 86400000 * 6;
+    const state: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: shownAt,
+      analyticsConsentRemindAt: remindAt,
+    };
+    expect(state.analyticsEnabled).toBeNull();
+    expect(state.analyticsConsentShownAt).toBe(shownAt);
+    expect(state.analyticsConsentRemindAt).toBe(remindAt);
+  });
+
+  it("accepts mixed state with analyticsEnabled true and remindAt set", () => {
+    const state: ConsentState = {
+      analyticsEnabled: true,
+      analyticsConsentShownAt: 1713800000000,
+      analyticsConsentRemindAt: 1714400000000,
+    };
+    expect(state.analyticsEnabled).toBe(true);
+    expect(state.analyticsConsentRemindAt).toBe(1714400000000);
+  });
+
+  it("has exactly the expected keys", () => {
+    const state: ConsentState = {
+      analyticsEnabled: null,
+      analyticsConsentShownAt: null,
+      analyticsConsentRemindAt: null,
+    };
+    const keys = Object.keys(state).sort();
+    expect(keys).toEqual(
+      ["analyticsConsentRemindAt", "analyticsConsentShownAt", "analyticsEnabled"].sort(),
+    );
+  });
+});

tests/unit/api/analytics-env.test.ts

@@ -0,0 +1,154 @@
+import { afterAll, beforeEach, describe, expect, it } from "vitest";
+
+describe("analytics env var validation", () => {
+  const originalEnv = { ...process.env };
+
+  // Keys that the Zod schema cares about -- we clear them before each test
+  // so defaults kick in unless explicitly set.
+  const analyticsKeys = [
+    "ANALYTICS_ENABLED",
+    "ANALYTICS_SAMPLE_RATE",
+    "POSTHOG_API_KEY",
+    "POSTHOG_HOST",
+    "SENTRY_DSN",
+  ];
+
+  beforeEach(() => {
+    for (const key of analyticsKeys) {
+      delete process.env[key];
+    }
+  });
+
+  afterAll(() => {
+    for (const key of Object.keys(process.env)) {
+      if (!(key in originalEnv)) delete process.env[key];
+    }
+    Object.assign(process.env, originalEnv);
+  });
+
+  // ── ANALYTICS_ENABLED ───────────────────────────────────────────────────
+
+  it("ANALYTICS_ENABLED defaults to true", async () => {
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    const env = loadEnv();
+    expect(env.ANALYTICS_ENABLED).toBe(true);
+  });
+
+  it("ANALYTICS_ENABLED='false' transforms to boolean false", async () => {
+    process.env.ANALYTICS_ENABLED = "false";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(loadEnv().ANALYTICS_ENABLED).toBe(false);
+  });
+
+  it("ANALYTICS_ENABLED='true' transforms to boolean true", async () => {
+    process.env.ANALYTICS_ENABLED = "true";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(loadEnv().ANALYTICS_ENABLED).toBe(true);
+  });
+
+  it("ANALYTICS_ENABLED rejects non-enum values", async () => {
+    process.env.ANALYTICS_ENABLED = "yes";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(() => loadEnv()).toThrow();
+  });
+
+  // ── ANALYTICS_SAMPLE_RATE ──────────────────────────────────────────────
+
+  it("ANALYTICS_SAMPLE_RATE defaults to 1.0", async () => {
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    const env = loadEnv();
+    expect(env.ANALYTICS_SAMPLE_RATE).toBe(1.0);
+  });
+
+  it("ANALYTICS_SAMPLE_RATE=0.5 parses correctly", async () => {
+    process.env.ANALYTICS_SAMPLE_RATE = "0.5";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(loadEnv().ANALYTICS_SAMPLE_RATE).toBe(0.5);
+  });
+
+  it("ANALYTICS_SAMPLE_RATE=0 is valid (no sampling)", async () => {
+    process.env.ANALYTICS_SAMPLE_RATE = "0";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(loadEnv().ANALYTICS_SAMPLE_RATE).toBe(0);
+  });
+
+  it("ANALYTICS_SAMPLE_RATE=1 is valid (full sampling)", async () => {
+    process.env.ANALYTICS_SAMPLE_RATE = "1";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(loadEnv().ANALYTICS_SAMPLE_RATE).toBe(1);
+  });
+
+  it("ANALYTICS_SAMPLE_RATE > 1 fails validation", async () => {
+    process.env.ANALYTICS_SAMPLE_RATE = "1.5";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(() => loadEnv()).toThrow();
+  });
+
+  it("ANALYTICS_SAMPLE_RATE < 0 fails validation", async () => {
+    process.env.ANALYTICS_SAMPLE_RATE = "-0.1";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(() => loadEnv()).toThrow();
+  });
+
+  it("ANALYTICS_SAMPLE_RATE=2 fails validation", async () => {
+    process.env.ANALYTICS_SAMPLE_RATE = "2";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(() => loadEnv()).toThrow();
+  });
+
+  // ── POSTHOG_API_KEY ────────────────────────────────────────────────────
+
+  it("POSTHOG_API_KEY defaults to the baked-in key", async () => {
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    const env = loadEnv();
+    expect(env.POSTHOG_API_KEY).toBe("phc_CVHjGivwWVzh76M5EjijTwP5LpiqWie3EbCzXU7w2Smy");
+  });
+
+  it("POSTHOG_API_KEY can be overridden with a custom value", async () => {
+    process.env.POSTHOG_API_KEY = "phc_custom_key_123";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(loadEnv().POSTHOG_API_KEY).toBe("phc_custom_key_123");
+  });
+
+  it("POSTHOG_API_KEY can be set to empty string", async () => {
+    process.env.POSTHOG_API_KEY = "";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(loadEnv().POSTHOG_API_KEY).toBe("");
+  });
+
+  // ── POSTHOG_HOST ───────────────────────────────────────────────────────
+
+  it("POSTHOG_HOST defaults to the PostHog US endpoint", async () => {
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    const env = loadEnv();
+    expect(env.POSTHOG_HOST).toBe("https://us.i.posthog.com");
+  });
+
+  it("POSTHOG_HOST can be overridden", async () => {
+    process.env.POSTHOG_HOST = "https://eu.posthog.com";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(loadEnv().POSTHOG_HOST).toBe("https://eu.posthog.com");
+  });
+
+  // ── SENTRY_DSN ─────────────────────────────────────────────────────────
+
+  it("SENTRY_DSN defaults to the baked-in DSN", async () => {
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    const env = loadEnv();
+    expect(env.SENTRY_DSN).toBe(
+      "https://2fd53fc3b3fdc59d02cac044a4f90b71@o4511263372738560.ingest.us.sentry.io/4511264620085248",
+    );
+  });
+
+  it("SENTRY_DSN can be overridden with a custom value", async () => {
+    process.env.SENTRY_DSN = "https://custom@sentry.io/999";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(loadEnv().SENTRY_DSN).toBe("https://custom@sentry.io/999");
+  });
+
+  it("SENTRY_DSN can be set to empty string to disable", async () => {
+    process.env.SENTRY_DSN = "";
+    const { loadEnv } = await import("../../../apps/api/src/lib/env.js");
+    expect(loadEnv().SENTRY_DSN).toBe("");
+  });
+});