mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
Five fixes to the PostHog event stream, from an audit of what we actually collect versus what's flowing in. Each one is test-first. ## What changed **Silenced the `$opt_in` noise.** `initAnalytics` called `opt_in_capturing()` on every page load to clear a stale opt-out flag, and posthog-js emits an `$opt_in` event on every call. That was 10k+ events a month (up to 55 per user) carrying no signal: analytics is on by default with an admin opt-out, so there is no per-user consent to record. Both call sites now pass `captureEventName: false`. **Split the onboarding survey out of `feedback_submitted`.** The onboarding usage survey rode the same event as real feedback, so about 93% of "feedback" was actually onboarding profiling. It now emits `onboarding_survey_submitted`, so feedback metrics mean feedback again. **Set `pipeline_executed.file_count`.** It was declared in the properties interface but never populated. A pure `pipelineExecutedProps` helper now derives it (batch size for a batch run, else 1) and is shared by the success and failure paths, which also drops a duplicated payload. **Tracked OIDC login failures.** All six OIDC callback failure branches bumped the Prometheus counter and wrote an audit log but never emitted `auth_login_failed`. A `recordOidcFailure` helper mirrors the password path. **Added `TELEMETRY.md`.** A contributor-facing event dictionary: every event, its properties, where it fires, and the privacy invariants, with the allowlists as source of truth. A drift test fails if any `ANALYTICS_EVENTS` value goes undocumented. I left the published telemetry guide (`apps/docs/guide/telemetry.md`) alone. It is high-level and still accurate, and editing it would pull in the 21-locale stale-gate for no gain. ## Verification - Unit (63 tests): `analytics-events`, `telemetry-doc-drift`, `api/analytics`, `web/analytics`, `worker.behavior` - Integration (41 tests): `oidc-auth`, `feedback` - Full typecheck across all 9 workspaces - Biome clean on the changed files All green locally.
96 lines
4.0 KiB
TypeScript
96 lines
4.0 KiB
TypeScript
import { ANALYTICS_EVENTS } from "@snapotter/shared";
|
|
import { describe, expect, it } from "vitest";
|
|
|
|
describe("ANALYTICS_EVENTS", () => {
|
|
it("has exactly 25 event keys", () => {
|
|
expect(Object.keys(ANALYTICS_EVENTS)).toHaveLength(25);
|
|
});
|
|
|
|
it("contains the expected keys", () => {
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("TOOL_USED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("TOOL_OPENED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("FILE_ADDED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("TOOL_STARTED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("TOOL_CLIENT_ERROR");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("RESULT_DOWNLOADED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("RESULT_SAVED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("SEARCH");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("PIPELINE_EXECUTED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("AI_BUNDLE_ACTION");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("AI_BUNDLE_PROMPTED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("BATCH_PROCESSED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("FEEDBACK_SUBMITTED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("ONBOARDING_SURVEY_SUBMITTED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("SPONSOR_CLICKED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("INSTANCE_STARTED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("EDITOR_OPENED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("EDITOR_TOOL_USED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("EDITOR_EXPORTED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("PIPELINE_OPENED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("PIPELINE_STEP_ADDED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("PIPELINE_SAVED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("PIPELINE_TEMPLATE_SELECTED");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("AUTH_LOGIN");
|
|
expect(ANALYTICS_EVENTS).toHaveProperty("AUTH_LOGIN_FAILED");
|
|
});
|
|
|
|
it("all event values are strings", () => {
|
|
for (const value of Object.values(ANALYTICS_EVENTS)) {
|
|
expect(typeof value).toBe("string");
|
|
}
|
|
});
|
|
|
|
it("TOOL_USED has the correct snake_case value", () => {
|
|
expect(ANALYTICS_EVENTS.TOOL_USED).toBe("tool_used");
|
|
});
|
|
|
|
it("SEARCH has the correct snake_case value", () => {
|
|
expect(ANALYTICS_EVENTS.SEARCH).toBe("search");
|
|
});
|
|
|
|
it("PIPELINE_EXECUTED has the correct snake_case value", () => {
|
|
expect(ANALYTICS_EVENTS.PIPELINE_EXECUTED).toBe("pipeline_executed");
|
|
});
|
|
|
|
it("AI_BUNDLE_ACTION has the correct snake_case value", () => {
|
|
expect(ANALYTICS_EVENTS.AI_BUNDLE_ACTION).toBe("ai_bundle_action");
|
|
});
|
|
|
|
it("FEEDBACK_SUBMITTED has the correct snake_case value", () => {
|
|
expect(ANALYTICS_EVENTS.FEEDBACK_SUBMITTED).toBe("feedback_submitted");
|
|
});
|
|
|
|
it("ONBOARDING_SURVEY_SUBMITTED has the correct snake_case value", () => {
|
|
expect(ANALYTICS_EVENTS.ONBOARDING_SURVEY_SUBMITTED).toBe("onboarding_survey_submitted");
|
|
});
|
|
|
|
it("INSTANCE_STARTED has the correct snake_case value", () => {
|
|
expect(ANALYTICS_EVENTS.INSTANCE_STARTED).toBe("instance_started");
|
|
});
|
|
|
|
it("all values follow snake_case convention", () => {
|
|
for (const value of Object.values(ANALYTICS_EVENTS)) {
|
|
expect(value).toMatch(/^[a-z][a-z0-9_]*$/);
|
|
}
|
|
});
|
|
|
|
it("is frozen (as const prevents mutation)", () => {
|
|
// as const produces a readonly object; Object.isFrozen checks runtime freezing.
|
|
// TypeScript enforces readonly at compile time, but at runtime the object
|
|
// defined with "as const" is a plain object unless explicitly frozen.
|
|
// We verify the values are stable by checking they haven't changed.
|
|
const snapshot = { ...ANALYTICS_EVENTS };
|
|
for (const key of Object.keys(snapshot)) {
|
|
expect(ANALYTICS_EVENTS[key as keyof typeof ANALYTICS_EVENTS]).toBe(
|
|
snapshot[key as keyof typeof snapshot],
|
|
);
|
|
}
|
|
});
|
|
|
|
it("all values are unique (no duplicate event names)", () => {
|
|
const values = Object.values(ANALYTICS_EVENTS);
|
|
const unique = new Set(values);
|
|
expect(unique.size).toBe(values.length);
|
|
});
|
|
});
|