feat(feedback): gate onboarding survey on first processing, add prompt lifecycle events (#615)

Defers the onboarding usage survey to the instance's first successful processing (the worker writes a one-time onboarding.firstProcessedAt marker and the overlay gates on it), so it reaches engaged users instead of first-landing visitors.

Replaces the two questions telemetry already answers (modality preference from tool_used, install method from instance_started) with what it can't infer: prior tool, self-host motivation, and discovery source.

Adds feedback_prompt_shown and feedback_prompt_dismissed on all five feedback surfaces (usage survey, per-job prompt, admin install card, global nav dialog, search-miss) so skip and completion rates are measurable, not just submissions. New survey strings translated into all 20 non-English locales.

tests/integration/platform/feedback.test.ts

@@ -134,7 +134,7 @@ describe("POST /api/v1/feedback", () => {
     );
   });
 
-  it("accepts an onboarding usage-survey submission", async () => {
+  it("accepts an onboarding usage-survey submission with the telemetry-blind answers", async () => {
     process.env.ANALYTICS_BAKED_OVERRIDE = "on";
     await refreshAnalyticsGate();
     const token = await loginAsAdmin(testApp.app);
@@ -148,7 +148,9 @@ describe("POST /api/v1/feedback", () => {
         surveyId: "onboarding-usage-v1",
         promptVariant: "onboarding-overlay-v1",
         usageType: "team_internal",
-        importantAreas: ["images", "pdf_docs"],
+        priorTool: "command_line",
+        selfHostMotivation: "privacy_control",
+        discoverySource: "github",
       },
     });
 
@@ -160,13 +162,15 @@ describe("POST /api/v1/feedback", () => {
         survey_id: "onboarding-usage-v1",
         prompt_variant: "onboarding-overlay-v1",
         usage_type: "team_internal",
-        important_areas: ["images", "pdf_docs"],
+        prior_tool: "command_line",
+        selfhost_motivation: "privacy_control",
+        discovery_source: "github",
       }),
       undefined,
     );
   });
 
-  it("accepts a persona-only onboarding submission with no important areas selected", async () => {
+  it("accepts a persona-only onboarding submission with only the usage type", async () => {
     process.env.ANALYTICS_BAKED_OVERRIDE = "on";
     await refreshAnalyticsGate();
     const token = await loginAsAdmin(testApp.app);
@@ -180,7 +184,6 @@ describe("POST /api/v1/feedback", () => {
         surveyId: "onboarding-usage-v1",
         promptVariant: "onboarding-overlay-v1",
         usageType: "personal",
-        importantAreas: [],
       },
     });
 
@@ -195,12 +198,26 @@ describe("POST /api/v1/feedback", () => {
       }),
       undefined,
     );
-    // captureFeedback is mocked, so it receives the route's raw properties: the
-    // empty importantAreas array is forwarded as-is. Dropping an empty
-    // important_areas before it reaches PostHog happens inside the real,
-    // unmocked cleanFeedbackProperties (analytics.ts), which this test bypasses.
-    const lastCall = captureFeedback.mock.calls.at(-1);
-    expect(lastCall?.[0].important_areas).toEqual([]);
+  });
+
+  it("rejects invalid onboarding survey answers", async () => {
+    const token = await loginAsAdmin(testApp.app);
+    const res = await testApp.app.inject({
+      method: "POST",
+      url: "/api/v1/feedback",
+      headers: { authorization: `Bearer ${token}` },
+      payload: {
+        source: "onboarding",
+        surveyId: "onboarding-usage-v1",
+        usageType: "personal",
+        priorTool: "carrier_pigeon",
+      },
+    });
+
+    expect(res.statusCode).toBe(400);
+    expect(JSON.parse(res.body).details).toContainEqual(
+      expect.objectContaining({ path: "priorTool" }),
+    );
   });
 
   it("drops identifying contact fields when contact consent is not checked", async () => {

tests/unit/api/analytics.test.ts

@@ -306,6 +306,9 @@ describe("captureFeedback", () => {
         survey_id: "onboarding-usage-v1",
         contact_ok: false,
         usage_type: "personal",
+        prior_tool: "command_line",
+        selfhost_motivation: "privacy_control",
+        discovery_source: "github",
       },
       "distinct-onboarding",
     );
@@ -314,6 +317,12 @@ describe("captureFeedback", () => {
       expect.objectContaining({
         distinctId: "distinct-onboarding",
         event: "onboarding_survey_submitted",
+        properties: expect.objectContaining({
+          usage_type: "personal",
+          prior_tool: "command_line",
+          selfhost_motivation: "privacy_control",
+          discovery_source: "github",
+        }),
       }),
     );
   });

tests/unit/api/settings-helpers.test.ts

@@ -5,6 +5,7 @@ import { db, schema } from "../../../apps/api/src/db/index.js";
 import {
   getSettingNumber,
   getSettingString,
+  setSettingIfAbsent,
   upsertSetting,
 } from "../../../apps/api/src/lib/settings-helpers.js";
 
@@ -117,3 +118,28 @@ describe("getSettingString", () => {
     expect(result).toBe("");
   });
 });
+
+describe("setSettingIfAbsent", () => {
+  it("writes the value when the key is absent", async () => {
+    const key = uniqueKey();
+    keysToClean.push(key);
+
+    await setSettingIfAbsent(key, "first");
+
+    const rows = await db.select().from(schema.settings).where(eq(schema.settings.key, key));
+    expect(rows).toHaveLength(1);
+    expect(rows[0].value).toBe("first");
+  });
+
+  it("keeps the first value and ignores later writes (first-write-wins)", async () => {
+    const key = uniqueKey();
+    keysToClean.push(key);
+
+    await setSettingIfAbsent(key, "first");
+    await setSettingIfAbsent(key, "second");
+
+    const rows = await db.select().from(schema.settings).where(eq(schema.settings.key, key));
+    expect(rows).toHaveLength(1);
+    expect(rows[0].value).toBe("first");
+  });
+});

tests/unit/shared/analytics-events.test.ts

@@ -2,8 +2,8 @@ import { ANALYTICS_EVENTS } from "@snapotter/shared";
 import { describe, expect, it } from "vitest";
 
 describe("ANALYTICS_EVENTS", () => {
-  it("has exactly 25 event keys", () => {
-    expect(Object.keys(ANALYTICS_EVENTS)).toHaveLength(25);
+  it("has exactly 27 event keys", () => {
+    expect(Object.keys(ANALYTICS_EVENTS)).toHaveLength(27);
   });
 
   it("contains the expected keys", () => {
@@ -32,6 +32,8 @@ describe("ANALYTICS_EVENTS", () => {
     expect(ANALYTICS_EVENTS).toHaveProperty("PIPELINE_TEMPLATE_SELECTED");
     expect(ANALYTICS_EVENTS).toHaveProperty("AUTH_LOGIN");
     expect(ANALYTICS_EVENTS).toHaveProperty("AUTH_LOGIN_FAILED");
+    expect(ANALYTICS_EVENTS).toHaveProperty("FEEDBACK_PROMPT_SHOWN");
+    expect(ANALYTICS_EVENTS).toHaveProperty("FEEDBACK_PROMPT_DISMISSED");
   });
 
   it("all event values are strings", () => {
@@ -68,6 +70,11 @@ describe("ANALYTICS_EVENTS", () => {
     expect(ANALYTICS_EVENTS.INSTANCE_STARTED).toBe("instance_started");
   });
 
+  it("feedback prompt lifecycle events have the correct snake_case values", () => {
+    expect(ANALYTICS_EVENTS.FEEDBACK_PROMPT_SHOWN).toBe("feedback_prompt_shown");
+    expect(ANALYTICS_EVENTS.FEEDBACK_PROMPT_DISMISSED).toBe("feedback_prompt_dismissed");
+  });
+
   it("all values follow snake_case convention", () => {
     for (const value of Object.values(ANALYTICS_EVENTS)) {
       expect(value).toMatch(/^[a-z][a-z0-9_]*$/);

tests/unit/web/feedback-visibility.test.ts

@@ -97,10 +97,14 @@ describe("shouldShowInstallFeedbackCard", () => {
 });
 
 describe("shouldShowUsageSurvey", () => {
-  it("shows only for admins after analytics config is loaded and enabled", () => {
+  // The survey now waits for the instance's first successful processing so we
+  // ask engaged users, not someone staring at an empty app on first landing.
+  const PROCESSED = { "onboarding.firstProcessedAt": "2026-01-14T00:00:00Z" };
+
+  it("shows only for admins after analytics is loaded and enabled, once a processing has completed", () => {
     expect(
       shouldShowUsageSurvey({
-        settings: {},
+        settings: PROCESSED,
         role: "admin",
         analyticsConfigLoaded: true,
         analyticsEnabled: true,
@@ -109,7 +113,7 @@ describe("shouldShowUsageSurvey", () => {
 
     expect(
       shouldShowUsageSurvey({
-        settings: {},
+        settings: PROCESSED,
         role: "user",
         analyticsConfigLoaded: true,
         analyticsEnabled: true,
@@ -118,7 +122,7 @@ describe("shouldShowUsageSurvey", () => {
 
     expect(
       shouldShowUsageSurvey({
-        settings: {},
+        settings: PROCESSED,
         role: "admin",
         analyticsConfigLoaded: false,
         analyticsEnabled: true,
@@ -127,7 +131,7 @@ describe("shouldShowUsageSurvey", () => {
 
     expect(
       shouldShowUsageSurvey({
-        settings: {},
+        settings: PROCESSED,
         role: "admin",
         analyticsConfigLoaded: true,
         analyticsEnabled: false,
@@ -135,10 +139,21 @@ describe("shouldShowUsageSurvey", () => {
     ).toBe(false);
   });
 
-  it("stays hidden after answering or permanently dismissing", () => {
+  it("stays hidden until the instance's first successful processing", () => {
     expect(
       shouldShowUsageSurvey({
-        settings: { "onboarding.usageSurvey.answeredAt": "2026-01-14T00:00:00Z" },
+        settings: {},
+        role: "admin",
+        analyticsConfigLoaded: true,
+        analyticsEnabled: true,
+      }),
+    ).toBe(false);
+  });
+
+  it("stays hidden after answering or permanently dismissing, even once processing has happened", () => {
+    expect(
+      shouldShowUsageSurvey({
+        settings: { ...PROCESSED, "onboarding.usageSurvey.answeredAt": "2026-01-14T00:00:00Z" },
         role: "admin",
         analyticsConfigLoaded: true,
         analyticsEnabled: true,
@@ -147,7 +162,7 @@ describe("shouldShowUsageSurvey", () => {
 
     expect(
       shouldShowUsageSurvey({
-        settings: { "onboarding.usageSurvey.dismissedAt": "2026-01-14T00:00:00Z" },
+        settings: { ...PROCESSED, "onboarding.usageSurvey.dismissedAt": "2026-01-14T00:00:00Z" },
         role: "admin",
         analyticsConfigLoaded: true,
         analyticsEnabled: true,

tests/unit/web/usage-survey-overlay.test.tsx

@@ -6,13 +6,15 @@ import { MemoryRouter } from "react-router-dom";
 import { afterEach, describe, expect, it, vi } from "vitest";
 
 const submitFeedback = vi.hoisted(() => vi.fn().mockResolvedValue({ ok: true, accepted: true }));
+const trackFeedbackPromptShown = vi.hoisted(() => vi.fn());
+const trackFeedbackPromptDismissed = vi.hoisted(() => vi.fn());
 const apiGet = vi.hoisted(() => vi.fn());
 const apiPut = vi.hoisted(() => vi.fn().mockResolvedValue({}));
 const useAuth = vi.hoisted(() => vi.fn());
 
 vi.mock("@/lib/feedback", async (importOriginal) => {
   const actual: Record<string, unknown> = await importOriginal();
-  return { ...actual, submitFeedback };
+  return { ...actual, submitFeedback, trackFeedbackPromptShown, trackFeedbackPromptDismissed };
 });
 
 vi.mock("@/lib/api", async (importOriginal) => {
@@ -30,9 +32,15 @@ vi.mock("@/stores/analytics-store", () => ({
 
 import { UsageSurveyOverlay } from "@/components/onboarding/usage-survey-overlay";
 
+// The worker writes this marker on the instance's first successful processing.
+// The survey is only eligible once it exists; without it the overlay stays hidden.
+const PROCESSED = { "onboarding.firstProcessedAt": "2026-01-01T00:00:00Z" };
+
 afterEach(() => {
   cleanup();
   submitFeedback.mockClear();
+  trackFeedbackPromptShown.mockClear();
+  trackFeedbackPromptDismissed.mockClear();
   apiGet.mockClear();
   apiPut.mockClear();
   useAuth.mockReset();
@@ -56,10 +64,21 @@ describe("UsageSurveyOverlay", () => {
     expect(screen.queryByText("How are you using SnapOtter?")).toBeNull();
   });
 
+  it("stays hidden until the instance's first processing has completed", async () => {
+    useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
+    apiGet.mockResolvedValue({ settings: {} });
+
+    renderOverlay();
+
+    await waitFor(() => expect(apiGet).toHaveBeenCalled());
+    expect(screen.queryByText("How are you using SnapOtter?")).toBeNull();
+    expect(trackFeedbackPromptShown).not.toHaveBeenCalled();
+  });
+
   it("renders nothing once already answered or dismissed", async () => {
     useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
     apiGet.mockResolvedValue({
-      settings: { "onboarding.usageSurvey.dismissedAt": "2026-01-01T00:00:00Z" },
+      settings: { ...PROCESSED, "onboarding.usageSurvey.dismissedAt": "2026-01-01T00:00:00Z" },
     });
 
     renderOverlay();
@@ -68,28 +87,28 @@ describe("UsageSurveyOverlay", () => {
     expect(screen.queryByText("How are you using SnapOtter?")).toBeNull();
   });
 
-  it("shows both questions for an admin instance that hasn't answered", async () => {
+  it("shows the telemetry-blind questions after processing and emits a shown event", async () => {
     useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
-    apiGet.mockResolvedValue({ settings: {} });
+    apiGet.mockResolvedValue({ settings: PROCESSED });
 
     renderOverlay();
 
     expect(await screen.findByText("How are you using SnapOtter?")).toBeDefined();
-    expect(screen.getByText("What matters most to you?")).toBeDefined();
+    expect(screen.getByText("What were you using before?")).toBeDefined();
+    expect(screen.getByText("Why self-host it?")).toBeDefined();
     expect(screen.getByRole("radio", { name: /Just me/ })).toBeDefined();
     expect(screen.getByRole("button", { name: "Continue" })).toBeDisabled();
+    await waitFor(() => expect(trackFeedbackPromptShown).toHaveBeenCalledWith("onboarding"));
   });
 
-  it("submits the selected answers and records the settings key", async () => {
+  it("submits only the usage type when nothing else is picked", async () => {
     useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
-    apiGet.mockResolvedValue({ settings: {} });
+    apiGet.mockResolvedValue({ settings: PROCESSED });
 
     renderOverlay();
     await screen.findByText("How are you using SnapOtter?");
 
     fireEvent.click(screen.getByRole("radio", { name: /Small team/ }));
-    fireEvent.click(screen.getByRole("button", { name: /Images/ }));
-    fireEvent.click(screen.getByRole("button", { name: /PDF\/docs/ }));
     fireEvent.click(screen.getByRole("button", { name: "Continue" }));
 
     await waitFor(() => {
@@ -98,7 +117,6 @@ describe("UsageSurveyOverlay", () => {
         surveyId: "onboarding-usage-v1",
         promptVariant: "onboarding-overlay-v1",
         usageType: "team_internal",
-        importantAreas: ["images", "pdf_docs"],
       });
     });
     expect(apiPut).toHaveBeenCalledWith("/v1/settings", {
@@ -106,17 +124,18 @@ describe("UsageSurveyOverlay", () => {
     });
   });
 
-  it("includes install method and friction area when the admin selects them", async () => {
+  it("includes prior tool, motivation, and discovery source when the admin selects them", async () => {
     useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
-    apiGet.mockResolvedValue({ settings: {} });
+    apiGet.mockResolvedValue({ settings: PROCESSED });
 
     renderOverlay();
     await screen.findByText("How are you using SnapOtter?");
 
     fireEvent.click(screen.getByRole("radio", { name: /Just me/ }));
-    fireEvent.click(screen.getByRole("radio", { name: "Built from source" }));
-    fireEvent.change(screen.getByLabelText("Hardest setup area"), {
-      target: { value: "docker" },
+    fireEvent.click(screen.getByRole("radio", { name: /Command line/ }));
+    fireEvent.click(screen.getByRole("radio", { name: /Privacy and data control/ }));
+    fireEvent.change(screen.getByLabelText(/How did you hear about us/), {
+      target: { value: "github" },
     });
     fireEvent.click(screen.getByRole("button", { name: "Continue" }));
 
@@ -126,36 +145,34 @@ describe("UsageSurveyOverlay", () => {
         surveyId: "onboarding-usage-v1",
         promptVariant: "onboarding-overlay-v1",
         usageType: "personal",
-        importantAreas: [],
-        installMethod: "source",
-        frictionArea: "docker",
+        priorTool: "command_line",
+        selfHostMotivation: "privacy_control",
+        discoverySource: "github",
       });
     });
   });
 
-  it("does not resubmit feedback if only the settings write failed on the first attempt", async () => {
+  it("dismissing writes the dismiss key, emits a dismissed event, and does not submit", async () => {
     useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
-    apiGet.mockResolvedValue({ settings: {} });
-    apiPut.mockRejectedValueOnce(new Error("network error")).mockResolvedValueOnce({});
+    apiGet.mockResolvedValue({ settings: PROCESSED });
 
     renderOverlay();
     await screen.findByText("How are you using SnapOtter?");
 
-    fireEvent.click(screen.getByRole("radio", { name: /Just me/ }));
-    fireEvent.click(screen.getByRole("button", { name: "Continue" }));
+    fireEvent.click(screen.getByRole("button", { name: "Don't ask again" }));
 
-    await waitFor(() => expect(apiPut).toHaveBeenCalledTimes(1));
-    expect(submitFeedback).toHaveBeenCalledTimes(1);
-
-    fireEvent.click(screen.getByRole("button", { name: "Continue" }));
-
-    await waitFor(() => expect(apiPut).toHaveBeenCalledTimes(2));
-    expect(submitFeedback).toHaveBeenCalledTimes(1);
+    await waitFor(() => {
+      expect(apiPut).toHaveBeenCalledWith("/v1/settings", {
+        "onboarding.usageSurvey.dismissedAt": expect.any(String),
+      });
+    });
+    expect(trackFeedbackPromptDismissed).toHaveBeenCalledWith("onboarding", "dont_ask_again");
+    expect(submitFeedback).not.toHaveBeenCalled();
   });
 
-  it("resubmits feedback if the answer changes after a failed settings write", async () => {
+  it("does not resubmit feedback if only the settings write failed on the first attempt", async () => {
     useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
-    apiGet.mockResolvedValue({ settings: {} });
+    apiGet.mockResolvedValue({ settings: PROCESSED });
     apiPut.mockRejectedValueOnce(new Error("network error")).mockResolvedValueOnce({});
 
     renderOverlay();
@@ -163,22 +180,19 @@ describe("UsageSurveyOverlay", () => {
 
     fireEvent.click(screen.getByRole("radio", { name: /Just me/ }));
     fireEvent.click(screen.getByRole("button", { name: "Continue" }));
-    await waitFor(() => expect(apiPut).toHaveBeenCalledTimes(1));
 
-    fireEvent.click(screen.getByRole("radio", { name: /Small team/ }));
+    await waitFor(() => expect(apiPut).toHaveBeenCalledTimes(1));
+    expect(submitFeedback).toHaveBeenCalledTimes(1);
+
     fireEvent.click(screen.getByRole("button", { name: "Continue" }));
 
     await waitFor(() => expect(apiPut).toHaveBeenCalledTimes(2));
-    expect(submitFeedback).toHaveBeenCalledTimes(2);
-    expect(submitFeedback).toHaveBeenNthCalledWith(
-      2,
-      expect.objectContaining({ usageType: "team_internal" }),
-    );
+    expect(submitFeedback).toHaveBeenCalledTimes(1);
   });
 
   it("stays visible and re-enables Continue if the feedback submission itself fails", async () => {
     useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
-    apiGet.mockResolvedValue({ settings: {} });
+    apiGet.mockResolvedValue({ settings: PROCESSED });
     submitFeedback.mockRejectedValueOnce(new Error("network error"));
 
     renderOverlay();
@@ -196,26 +210,9 @@ describe("UsageSurveyOverlay", () => {
     expect(apiPut).not.toHaveBeenCalled();
   });
 
-  it("dismissing writes the dismiss key without submitting feedback", async () => {
-    useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
-    apiGet.mockResolvedValue({ settings: {} });
-
-    renderOverlay();
-    await screen.findByText("How are you using SnapOtter?");
-
-    fireEvent.click(screen.getByRole("button", { name: "Don't ask again" }));
-
-    await waitFor(() => {
-      expect(apiPut).toHaveBeenCalledWith("/v1/settings", {
-        "onboarding.usageSurvey.dismissedAt": expect.any(String),
-      });
-    });
-    expect(submitFeedback).not.toHaveBeenCalled();
-  });
-
   it("ignores a second dismiss click while the first write is in flight", async () => {
     useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
-    apiGet.mockResolvedValue({ settings: {} });
+    apiGet.mockResolvedValue({ settings: PROCESSED });
     let resolveApiPut: (() => void) | undefined;
     apiPut.mockImplementationOnce(
       () =>
@@ -244,7 +241,7 @@ describe("UsageSurveyOverlay", () => {
 
   it("renders nothing when the admin must still change their password", async () => {
     useAuth.mockReturnValue({ role: "admin", mustChangePassword: true });
-    apiGet.mockResolvedValue({ settings: {} });
+    apiGet.mockResolvedValue({ settings: PROCESSED });
 
     renderOverlay();
 
@@ -255,7 +252,7 @@ describe("UsageSurveyOverlay", () => {
 
   it("renders nothing on the change-password route even if mustChangePassword is stale-false", async () => {
     useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
-    apiGet.mockResolvedValue({ settings: {} });
+    apiGet.mockResolvedValue({ settings: PROCESSED });
 
     renderOverlay("/change-password");
 
@@ -266,7 +263,7 @@ describe("UsageSurveyOverlay", () => {
 
   it("renders nothing on the privacy policy route", async () => {
     useAuth.mockReturnValue({ role: "admin", mustChangePassword: false });
-    apiGet.mockResolvedValue({ settings: {} });
+    apiGet.mockResolvedValue({ settings: PROCESSED });
 
     renderOverlay("/privacy");