diff --git a/drizzle/0007_workflow_status_canceled.sql b/drizzle/0007_workflow_status_canceled.sql new file mode 100644 index 0000000..f24166c --- /dev/null +++ b/drizzle/0007_workflow_status_canceled.sql @@ -0,0 +1,3 @@ +-- WorkflowRepository.cancelInstance writes 'canceled', which the enum never +-- had, so every cancel failed with invalid_text_representation. +ALTER TYPE "public"."workflow_status" ADD VALUE IF NOT EXISTS 'canceled'; diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 9b06cd4..eb06009 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1790600000000, "tag": "0006_message_logs_redundant_indexes", "breakpoints": true + }, + { + "idx": 7, + "version": "7", + "when": 1790700000000, + "tag": "0007_workflow_status_canceled", + "breakpoints": true } ] } diff --git a/packages/mcp/src/tools.ts b/packages/mcp/src/tools.ts index a380f2f..bf24553 100644 --- a/packages/mcp/src/tools.ts +++ b/packages/mcp/src/tools.ts @@ -302,24 +302,17 @@ export function registerTools(server: McpServer, api: NotifkitApi): void { if (seen.has(target)) continue; seen.add(target); - let userRecord: Record; - if (channel === "email") { - userRecord = { id: `email:${target}`, email: target }; - } else if (channel === "sms") { - userRecord = { id: `phone:${target}`, phone: target }; - } else if (channel === "push") { - userRecord = { id: `push:${target}`, pushToken: target }; - } else if (channel === "telegram") { - userRecord = { id: `telegram:${target}` }; - } else if (channel === "discord") { - userRecord = { id: `discord:${target}` }; - } else if (channel === "whatsapp") { - userRecord = { id: `whatsapp:${target}`, phone: target }; - } else if (channel === "slack") { - userRecord = { id: `slack:${target}` }; - } else { - userRecord = { id: `webhook:${target}` }; - } + // Every channel needs a contact on that channel to be reached. Chat + // channels used to get a bare user record, and WhatsApp a phone + // number filed as an SMS contact, so those campaigns sent nothing. + const userRecord: Record = + channel === "email" + ? { id: `email:${target}`, email: target } + : channel === "sms" + ? { id: `phone:${target}`, phone: target } + : channel === "push" + ? { id: `push:${target}`, pushToken: target } + : { id: `${channel}:${target}`, contacts: [{ channel, target }] }; recipients.push(userRecord); } diff --git a/packages/provider-discord/src/index.ts b/packages/provider-discord/src/index.ts index 75a6192..217f2d7 100644 --- a/packages/provider-discord/src/index.ts +++ b/packages/provider-discord/src/index.ts @@ -66,7 +66,11 @@ export class DiscordTransport implements Transport { const subject = content.subject; // Discord messages have no subject line of their own, so a template that // renders one is bolded onto the first line rather than dropped. - const text = subject ? `**${subject}**\n${body}` : body; + // Either part may be missing: a subject-only template must not post + // "undefined" underneath it. + const text = [subject ? `**${subject}**` : undefined, body] + .filter((part) => typeof part === "string" && part) + .join("\n"); // A template may carry its own sender, so one webhook can post as // different bots for different alert types. diff --git a/packages/provider-resend/src/index.ts b/packages/provider-resend/src/index.ts index e777ef5..5ed8c8b 100644 --- a/packages/provider-resend/src/index.ts +++ b/packages/provider-resend/src/index.ts @@ -73,7 +73,10 @@ export class ResendTransport implements Transport { from, to, subject: subject ?? "Notification", - html: htmlBody ?? `

${body}

`, + // A text-only template gets an HTML part built from its text. The text + // was rendered unescaped (it is plain text), so it is escaped here — + // otherwise a variable holding markup becomes live HTML in the email. + html: htmlBody ?? (typeof body === "string" ? textToHtml(body) : undefined), text: body, ...(replyTo ? { replyTo } : {}), ...(headers && Object.keys(headers).length > 0 ? { headers } : {}), @@ -249,3 +252,13 @@ export class ResendTransport implements Transport { return []; } } + +function textToHtml(text: string): string { + const escaped = text + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); + return `

${escaped.replace(/\r?\n/g, "
")}

`; +} diff --git a/packages/provider-telegram/src/index.ts b/packages/provider-telegram/src/index.ts index 2d94486..91cfd66 100644 --- a/packages/provider-telegram/src/index.ts +++ b/packages/provider-telegram/src/index.ts @@ -55,7 +55,12 @@ export class TelegramTransport implements Transport { const subject = content.subject; // Telegram messages have no subject line of their own, so a template // that renders one gets folded into the first line rather than dropped. - const text = subject ? `${subject}\n\n${body}` : body; + // Either part may be missing: a subject-only template must not send the + // word "undefined" underneath it. + const text = [subject, body].filter((part) => typeof part === "string" && part).join("\n\n"); + if (!text) { + return { success: false, error: "No message text on task" }; + } const parseMode = typeof content.parseMode === "string" ? content.parseMode : this.parseMode; try { diff --git a/src/client.ts b/src/client.ts index f33a5d3..4ba65e0 100644 --- a/src/client.ts +++ b/src/client.ts @@ -22,6 +22,7 @@ import type { SuppressionRecord, ProjectRecord, ProjectKeyRecord, + CreatedProjectKeyRecord, SystemHealthRecord, SystemMetricsRecord, DLQMessageRecord, @@ -61,10 +62,17 @@ export class NotifkitClient { return undefined as T; } - const data = await res.json(); + const text = await res.text(); + let data: unknown; + try { + data = text ? JSON.parse(text) : undefined; + } catch { + // Not JSON: an HTML error page, or a proxy in front of the API. + data = undefined; + } if (!res.ok) { - const errorMsg = - (data as any).message || (data as any).error || `Request failed with status ${res.status}`; + const body = data as { message?: string; error?: string } | undefined; + const errorMsg = body?.message || body?.error || `Request failed with status ${res.status}`; throw new Error(errorMsg); } return data as T; @@ -285,7 +293,7 @@ export class NotifkitClient { async createProjectKey( id: string, input?: { role?: "admin" | "read_only" }, - ): Promise { + ): Promise { return this.request(`/v1/projects/${encodeURIComponent(id)}/keys`, "POST", input || {}); } @@ -472,7 +480,9 @@ export class NotifkitClient { } /** Replay a dead-letter queue message. */ - async replayDLQMessage(id: string): Promise<{ success: boolean; replayedId: string }> { + async replayDLQMessage( + id: string, + ): Promise<{ success: boolean; replayedId: string; stream: string }> { return this.request("/v1/dlq/replay", "POST", { id }); } diff --git a/src/contracts/events/notification-ai-pending.ts b/src/contracts/events/notification-ai-pending.ts index d04ba6b..003b1cd 100644 --- a/src/contracts/events/notification-ai-pending.ts +++ b/src/contracts/events/notification-ai-pending.ts @@ -14,5 +14,17 @@ export const NotificationAiPendingPayloadSchema = z.object({ aiPrompts: z.record(z.string(), z.string()), scheduledAt: z.string().datetime().optional(), fallbackChain: z.array(NotificationChannelSchema).optional(), + /** + * The task id the engine assigned. Stable across replays, so a re-run of + * this event dispatches the same task — which delivery's lease then sends + * once — rather than a fresh one. + */ + taskId: z.string().min(1).optional(), + /** The contact address the engine resolved and cleared against suppressions. */ + destination: z.string().min(1).optional(), + /** Headers for the provider, e.g. one-click unsubscribe. */ + deliveryHeaders: z.record(z.string(), z.string()).optional(), + campaignId: z.string().min(1).max(128).optional(), + workflowInstanceId: z.string().uuid().optional(), }); export type NotificationAiPendingPayload = z.infer; diff --git a/src/contracts/events/notification-dispatched.ts b/src/contracts/events/notification-dispatched.ts index e305277..6ccd18b 100644 --- a/src/contracts/events/notification-dispatched.ts +++ b/src/contracts/events/notification-dispatched.ts @@ -45,8 +45,15 @@ export const NotificationDispatchedPayloadSchema = z.object({ deliveryOptions: DeliveryOptionsSchema, fallbackChain: z.array(NotificationChannelSchema).optional(), throttleAttemptCount: z.number().int().nonnegative().optional(), + /** + * Provider attempts that already failed. Delivery re-queues a failed send + * through the scheduler until this reaches `deliveryOptions.maxAttempts`. + */ + deliveryAttemptCount: z.number().int().nonnegative().optional(), /** Campaign this message belongs to, carried from the originating request. */ campaignId: z.string().min(1).max(128).optional(), + /** Workflow instance whose step sent this, recorded on the delivery log. */ + workflowInstanceId: z.string().uuid().optional(), }); /** A `notification.dispatched` task as it travels on the outbound stream. */ export type DispatchedTaskPayload = z.infer; diff --git a/src/contracts/events/notification-enriched.ts b/src/contracts/events/notification-enriched.ts index cbfd547..3e638b9 100644 --- a/src/contracts/events/notification-enriched.ts +++ b/src/contracts/events/notification-enriched.ts @@ -50,5 +50,13 @@ export const NotificationEnrichedPayloadSchema = z.object({ destination: z.string().min(1).optional(), /** Campaign this message belongs to, carried from the originating request. */ campaignId: z.string().min(1).max(128).optional(), + /** Workflow instance whose step sent this, carried to the delivery log. */ + workflowInstanceId: z.string().uuid().optional(), + /** + * Set on delivery's channel fallback. The notification was counted against + * the user's throttle when it was first sent; counting it again per channel + * it falls back through would drop the very send meant to reach them. + */ + throttleCounted: z.boolean().optional(), }); export type NotificationEnrichedPayload = z.infer; diff --git a/src/contracts/events/notification-requested.ts b/src/contracts/events/notification-requested.ts index 88a5930..bffe9ae 100644 --- a/src/contracts/events/notification-requested.ts +++ b/src/contracts/events/notification-requested.ts @@ -32,5 +32,7 @@ export const NotificationRequestedPayloadSchema = z.object({ * reported on afterwards. Carried unchanged to the delivery log. */ campaignId: z.string().min(1).max(128).optional(), + /** Set by the workflow worker on a step's notify, carried to the delivery log. */ + workflowInstanceId: z.string().uuid().optional(), }); export type NotificationRequestedPayload = z.infer; diff --git a/src/contracts/sdk.ts b/src/contracts/sdk.ts index d22509a..3a744d5 100644 --- a/src/contracts/sdk.ts +++ b/src/contracts/sdk.ts @@ -110,12 +110,12 @@ export const TemplateSchema = z.object({ content: z.record(z.string(), z.unknown()), aiPrompts: z.record(z.string(), z.string()).optional(), }); -export type TemplateInput = z.infer; +export type TemplateInput = z.input; export const SyncTemplatesSchema = z.object({ templates: z.array(TemplateSchema).min(1), }); -export type SyncTemplatesInput = z.infer; +export type SyncTemplatesInput = z.input; // ─── notify() ───────────────────────────────────────────────────────────────── @@ -216,7 +216,7 @@ export const TriggerWorkflowSchema = z.object({ input: z.record(z.string(), z.unknown()).optional(), user: z.union([z.string().min(1), InlineUserSchema]).optional(), }); -export type TriggerWorkflowInput = z.infer; +export type TriggerWorkflowInput = z.input; // ─── Events ─────────────────────────────────────────────────────────────────── @@ -241,6 +241,12 @@ export const WorkflowStepSchema = z.discriminatedUnion("action", [ options: z .object({ timeout: z.string().optional(), // e.g. "24h" + /** + * Fields the event must carry to wake this instance, e.g. + * `{ userId: "u1" }`. Without it any event of that name in the project + * wakes every instance waiting on it. + */ + match: z.record(z.string(), z.unknown()).optional(), }) .optional(), }), @@ -251,7 +257,7 @@ export const CreateWorkflowSchema = z.object({ name: z.string().min(1), steps: z.array(WorkflowStepSchema).min(1), }); -export type CreateWorkflowInput = z.infer; +export type CreateWorkflowInput = z.input; // ─── Projects ───────────────────────────────────────────────────────────────── @@ -291,7 +297,8 @@ export interface UserResponse extends UserProfileResponse { export interface UserDetailResponse extends UserResponse { contacts: UserContactResponse[]; - recentLogs?: unknown[]; + /** The 50 most recent delivery-log rows for this user's addresses. */ + logs: NotificationLogRecord[]; } export interface WorkflowDefinitionRecord { @@ -302,30 +309,53 @@ export interface WorkflowDefinitionRecord { updatedAt?: string; } -export interface WorkflowInstanceRecord { +export interface WorkflowStepRecord { id: string; - name: string; - status: string; - currentStepIndex: number; - input?: Record; - output?: Record; + instanceId: string; + stepIndex: string; + action: "notify" | "wait" | "waitForEvent" | "run" | string; + output: unknown; + error: string | null; createdAt: string; - updatedAt: string; } +export interface WorkflowWaiterRecord { + id: string; + instanceId: string; + eventName: string; + matchCriteria: Record; + expiresAt: string; + createdAt: string; +} + +export interface WorkflowInstanceRecord { + id: string; + projectId: string; + name: string; + status: "pending" | "running" | "completed" | "failed" | "canceled"; + input?: Record | null; + createdAt: string; + updatedAt: string; + steps: WorkflowStepRecord[]; + waiters: WorkflowWaiterRecord[]; +} + +/** One row of the delivery log, as the API returns it. */ export interface NotificationLogRecord { id: string; - notificationId: string; - taskId?: string; + projectId: string; + taskId: string; + providerMessageId: string | null; + templateId: string | null; + workflowInstanceId: string | null; channel: string; - target?: string; + attempt: number; + /** `dispatched`, `attempt`, or an engagement event such as `opened`. */ + kind: string; status: string; - templateId?: string; - campaign?: string; - createdAt: string; - dispatchedAt?: string; - error?: string; - metadata?: Record; + campaignId: string | null; + metadata: Record | null; + timestamp: string; } export interface TemplateRecordResponse { @@ -338,9 +368,12 @@ export interface TemplateRecordResponse { export interface SuppressionRecord { id: string; + projectId: string; channel: string; target: string; reason: string; + source: string | null; + taskId: string | null; createdAt: string; } @@ -353,30 +386,45 @@ export interface ProjectRecord { createdAt: string; } +/** A key as listed: the secret itself is only ever returned once, on creation. */ export interface ProjectKeyRecord { id: string; - apiKey: string; - role: string; + role: "admin" | "read_only"; createdAt: string; } +export interface CreatedProjectKeyRecord { + id: string; + apiKey: string; + role: "admin" | "read_only"; +} + export interface SystemHealthRecord { - status: string; - timestamp: string; - redis?: { status: string }; - db?: { status: string }; - workers?: Record; + status: "healthy" | "degraded"; + redis: { ok: boolean; latencyMs: number }; + db: { ok: boolean; latencyMs: number }; + /** Each worker's last heartbeat, or `{ status: "unknown" }` when none arrived. */ + workers: Record>; } export interface SystemMetricsRecord { - queues: Record; - throughput?: Record; + /** Unconsumed backlog per stream. Empty for project-scoped callers. */ + streams: Record; + deliveryStats: { + total: number; + delivered: number; + failed: number; + successRate: number; + }; } export interface DLQMessageRecord { id: string; - stream: string; + eventType: string; payload: Record; - error?: string; + /** Why the message was dead-lettered. */ + error: string; timestamp: string; + /** The stream it failed on, where a replay puts it back. */ + originalStream?: string; } diff --git a/src/db/schema.ts b/src/db/schema.ts index 6952994..4f8a1cc 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -270,6 +270,7 @@ export const workflowStatusEnum = pgEnum("workflow_status", [ "running", "completed", "failed", + "canceled", ]); export const workflowDefinitions = pgTable( diff --git a/src/queue/index.ts b/src/queue/index.ts index df96033..bb88a4b 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -96,12 +96,14 @@ export class StreamProducer { } async publish(partial: Omit): Promise { - const event: StreamEvent = { + return this.enqueue({ ...partial, id: crypto.randomUUID(), timestamp: new Date().toISOString(), - }; + }); + } + private enqueue(event: StreamEvent): Promise { return new Promise((resolve, reject) => { this.pendingQueue.push({ event, resolve, reject }); if (this.pendingQueue.length >= 100) { @@ -209,13 +211,15 @@ export class StreamProducer { ): Promise<{ messageIds: string[]; eventIds: string[] }> { if (partials.length === 0) return { messageIds: [], eventIds: [] }; + // The ids returned must be the ids the events are written with: callers + // record them as notification ids. const eventIds: string[] = []; const promises: Promise[] = []; for (const partial of partials) { const id = crypto.randomUUID(); eventIds.push(id); - promises.push(this.publish(partial)); + promises.push(this.enqueue({ ...partial, id, timestamp: new Date().toISOString() })); } const messageIds = await Promise.all(promises); @@ -472,7 +476,7 @@ export class StreamConsumer { * against this consumer's stream so a replay lands where it would have been * read from. */ - async deadLetter(event: StreamEvent): Promise { + async deadLetter(event: StreamEvent, reason?: string): Promise { if (!this.dlqStream) { this.logger?.error( { eventId: event.id }, @@ -486,12 +490,17 @@ export class StreamConsumer { "data", JSON.stringify({ ...event, - dlq: { originalStream: this.streams[0]!, ackedAt: new Date().toISOString() }, + dlq: { originalStream: this.streams[0]!, ackedAt: new Date().toISOString(), reason }, }), ); } - async nack(messageId: string, event: StreamEvent, stream?: string): Promise { + async nack( + messageId: string, + event: StreamEvent, + stream?: string, + reason?: string, + ): Promise { const s = stream ?? this.streams[0]!; if (this.dlqStream) { // Sequential rather than pipelined, and in this order: the ack is what @@ -506,7 +515,7 @@ export class StreamConsumer { "data", JSON.stringify({ ...event, - dlq: { originalStream: s, ackedAt: new Date().toISOString() }, + dlq: { originalStream: s, ackedAt: new Date().toISOString(), reason }, }), ); diff --git a/src/rate-limiter/index.ts b/src/rate-limiter/index.ts index f06eee8..c611184 100644 --- a/src/rate-limiter/index.ts +++ b/src/rate-limiter/index.ts @@ -308,24 +308,25 @@ export class ApiRateLimiter { } // 2. Slow Path: tokens are depleted, block on the in-flight lease or start one. - // Multiple concurrent requests racing here all join the *same* promise so - // only one Redis round trip happens. After it resolves each waiter tries to - // claim exactly one token — if they lose the race they're rate-limited. - if (!state.inFlightLease) { - this.refill(projectId, state, limitRpm, nowMs, currentBucket).catch(() => {}); - } + // Concurrent requests racing here all join the *same* promise so one Redis + // round trip serves them. A lease only grants `batchSize` tokens, so a + // burst larger than that leases again rather than rejecting requests the + // window still has room for: a request is refused only once Redis itself + // reports the window spent. + for (;;) { + if (!state.inFlightLease) { + this.refill(projectId, state, limitRpm, nowMs, currentBucket).catch(() => {}); + } + const granted = state.inFlightLease ? await state.inFlightLease : 0; - if (state.inFlightLease) { - await state.inFlightLease; + // Only claim a token if the lease came back for the same bucket we waited on. + if (state.bucketIndex !== currentBucket) return false; + if (state.tokens > 0) { + state.tokens--; + return true; + } + if (granted <= 0) return false; } - - // Only claim a token if the lease came back for the same bucket we waited on. - if (state.bucketIndex === currentBucket && state.tokens > 0) { - state.tokens--; - return true; - } - - return false; } private async refill( diff --git a/src/repositories/index.ts b/src/repositories/index.ts index 5d727e7..e02eac8 100644 --- a/src/repositories/index.ts +++ b/src/repositories/index.ts @@ -22,6 +22,8 @@ import { adminUsers, } from "@/db/schema.js"; +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + // ─── Domain types ─────────────────────────────────────────────────────────── export interface UserProfile { @@ -310,6 +312,12 @@ export class UserRepository { ): Promise { if (usersList.length === 0) return; + // ON CONFLICT DO UPDATE rejects the whole statement when two proposed rows + // hit the same key, so an import naming one user twice failed outright. + // The last occurrence wins, as it would had they been sent one by one. + const uniqueUsers = [...new Map(usersList.map((u) => [u.userId, u])).values()]; + const winners = new Set(uniqueUsers); + let attempts = 0; while (attempts < 3) { try { @@ -317,7 +325,7 @@ export class UserRepository { await tx .insert(users) .values( - usersList.map((u) => ({ + uniqueUsers.map((u) => ({ projectId, externalId: u.userId, attributes: { @@ -381,7 +389,12 @@ export class UserRepository { } } - if (u.preferences?.quietHours && u.preferences.quietHours.length > 0) { + // Quiet hours are replaced, not merged, so only the winning row counts. + if ( + winners.has(u) && + u.preferences?.quietHours && + u.preferences.quietHours.length > 0 + ) { for (const window of u.preferences.quietHours) { quietHoursInserts.push({ userId: internalId, @@ -438,7 +451,7 @@ export class UserRepository { }); } - const usersWithQuietHours = usersList.filter( + const usersWithQuietHours = uniqueUsers.filter( (u) => u.preferences?.quietHours !== undefined, ); const internalIdsToClearQuietHours = usersWithQuietHours @@ -485,7 +498,8 @@ export class UserRepository { const attrs = { language: patch.language ?? existing.language, timezone: patch.timezone ?? existing.timezone, - email: patch.email ?? existing.email, + // `null` clears the email; only an absent field keeps the old one. + email: patch.email !== undefined ? patch.email : existing.email, }; await this.db.transaction(async (tx) => { @@ -588,11 +602,14 @@ export class UserRepository { ): Promise<{ users: UserProfile[]; nextCursor: string | null }> { const conditions = [eq(users.projectId, projectId)]; - if (cursor) { - const cursorDate = new Date(parseInt(cursor, 10)); - if (!isNaN(cursorDate.getTime())) { - conditions.push(drizzleSql`${users.createdAt} < ${cursorDate.toISOString()}`); - } + // The cursor is the id of the last row returned. Paging on created_at + // alone skipped rows: a bulk import writes every user in one transaction, + // so they share one created_at, and the millisecond cursor could not tell + // them apart. (created_at, id) is unique and ordered. + if (cursor && UUID_RE.test(cursor)) { + conditions.push( + drizzleSql`(${users.createdAt}, ${users.id}) < (SELECT c.created_at, c.id FROM ${users} c WHERE c.id = ${cursor})`, + ); } if (filters?.language) { @@ -626,7 +643,7 @@ export class UserRepository { .select() .from(users) .where(and(...conditions)) - .orderBy(desc(users.createdAt)) + .orderBy(desc(users.createdAt), desc(users.id)) .limit(limit); const items = rows.map((r) => { @@ -640,8 +657,7 @@ export class UserRepository { }; }); - const nextCursor = - items.length === limit ? items[items.length - 1]!.createdAt.toString() : null; + const nextCursor = rows.length === limit ? rows[rows.length - 1]!.id : null; return { users: items, nextCursor }; } @@ -794,6 +810,25 @@ export class ContactRepository { ), ); + // Per-address topic opt-outs (a user who wants marketing at home but not at + // work). Without them the pipeline cannot honour an opt-out it accepted. + const topicsByContact = new Map>(); + if (rows.length > 0) { + const topicRows = await this.db + .select() + .from(contactTopicPreferences) + .where( + inArray( + contactTopicPreferences.contactId, + rows.map((r) => r.id), + ), + ); + for (const t of topicRows) { + if (!topicsByContact.has(t.contactId)) topicsByContact.set(t.contactId, {}); + topicsByContact.get(t.contactId)![t.topic] = t.enabled; + } + } + const fetchedByUser = new Map(); for (const id of missingIds) { fetchedByUser.set(id, []); @@ -801,12 +836,13 @@ export class ContactRepository { for (const row of rows) { const contacts = fetchedByUser.get(row.userId) ?? []; + const topics = topicsByContact.get(row.id); contacts.push({ id: row.id, userId: row.userId, channel: row.channel as ContactChannel, target: row.target, - preferences: {}, + preferences: topics ? { topics } : {}, active: row.enabled, }); fetchedByUser.set(row.userId, contacts); @@ -897,7 +933,29 @@ export class ContactRepository { const idMap = new Map(internalUserIdRows.map((r) => [r.externalId, r.id])); - const validContacts = contactsList.filter((c) => idMap.has(c.userId)); + // The same address can arrive twice — as `email` and again in `contacts` + // — and ON CONFLICT DO UPDATE rejects a statement that touches one row + // twice. Keep one entry per address, merging their topic preferences. + const byAddress = new Map(); + for (const c of contactsList) { + if (!idMap.has(c.userId)) continue; + const key = `${c.userId}\u0000${c.channel}\u0000${c.target}`; + const seen = byAddress.get(key); + byAddress.set( + key, + seen + ? { + ...c, + preferences: { + ...seen.preferences, + ...c.preferences, + topics: { ...seen.preferences?.topics, ...c.preferences?.topics }, + }, + } + : c, + ); + } + const validContacts = [...byAddress.values()]; if (validContacts.length === 0) return; const inserted = await this.db @@ -1118,6 +1176,7 @@ export class ProjectRepository { await tx.delete(suppressions).where(eq(suppressions.projectId, id)); await tx.delete(messageLogs).where(eq(messageLogs.projectId, id)); await tx.delete(workflowInstances).where(eq(workflowInstances.projectId, id)); + await tx.delete(workflowDefinitions).where(eq(workflowDefinitions.projectId, id)); const result = await tx.delete(projects).where(eq(projects.id, id)).returning(); return result.length > 0; @@ -1234,7 +1293,7 @@ export class WorkflowRepository { return await this.db.transaction(async (tx) => { const result = await tx .update(workflowInstances) - .set({ status: "canceled" as any }) + .set({ status: "canceled" }) .where( and( eq(workflowInstances.id, instanceId), diff --git a/src/services/ai/main.ts b/src/services/ai/main.ts index 843cc68..51fac6e 100644 --- a/src/services/ai/main.ts +++ b/src/services/ai/main.ts @@ -1,4 +1,3 @@ -import { randomUUID } from "node:crypto"; import { loadEnv, readBaseConfig } from "@/index.js"; import type { Logger } from "@/index.js"; import type { RedisClient, Redis } from "@/index.js"; @@ -31,6 +30,7 @@ import { createStreamConsumer, createWorkerRuntime, shutdownWorker, + uniqueConsumerId, } from "@/workers/bootstrap.js"; // ─── Bootstrap ───────────────────────────────────────────────────────────── @@ -128,11 +128,22 @@ export class AiWorker extends BaseWorker { const pending = payloadResult.data as NotificationAiPendingPayload; - // Idempotency - const idempotencyKey = `${pending.enrichedEventId}:${pending.recipientId}:${pending.channel}:ai`; + // Stable across replays: the engine's task id when it sent one. A fresh id + // per attempt made a replay after a crash a second, unrelated task that + // delivery's lease could not recognise — the recipient got it twice. + const taskId = + pending.taskId ?? `${pending.enrichedEventId}:${pending.recipientId}:${pending.channel}`; + + // Idempotency. The lease has to outlast every generation this message may + // run, or a second worker takes it over mid-call and bills the prompts again. + const idempotencyKey = `${taskId}:ai`; + const aiConfig = getAiConfig(); + const maxPrompts = aiConfig.maxPromptsPerNotification ?? AI_DEFAULTS.maxPromptsPerNotification; + const leaseSeconds = + Math.ceil(((aiConfig.timeoutMs ?? AI_DEFAULTS.timeoutMs) * maxPrompts) / 1000) + 30; const lease = this.idempotency.acquireLease - ? await this.idempotency.acquireLease(idempotencyKey, 30) - : (await this.idempotency.checkAndMark(idempotencyKey, 30)) + ? await this.idempotency.acquireLease(idempotencyKey, leaseSeconds) + : (await this.idempotency.checkAndMark(idempotencyKey, leaseSeconds)) ? "acquired" : "locked"; @@ -155,8 +166,6 @@ export class AiWorker extends BaseWorker { // Execute AI prompts. Each key is a separate billed model call, so the // count is capped rather than being driven by whatever the caller sent. const promptEntries = Object.entries(pending.aiPrompts); - const maxPrompts = - getAiConfig().maxPromptsPerNotification ?? AI_DEFAULTS.maxPromptsPerNotification; if (promptEntries.length > maxPrompts) { this.logger.warn( { messageId: message.id, requested: promptEntries.length, maxPrompts }, @@ -178,9 +187,9 @@ export class AiWorker extends BaseWorker { const rendered = renderWithTemplate(dbTemplate, finalVars); - const taskId = randomUUID(); const destination = - pending.channel === "email" + pending.destination ?? + (pending.channel === "email" ? pending.recipient.email : pending.channel === "sms" || pending.channel === "whatsapp" ? pending.recipient.phone @@ -194,7 +203,7 @@ export class AiWorker extends BaseWorker { ? pending.recipient.slack : pending.channel === "push" ? (pending.recipient.pushTokens?.[0] ?? pending.recipient.pushToken) - : undefined; + : undefined); const resolvedDestination = destination ?? (pending.channel === "push" ? undefined : pending.recipientId); @@ -214,8 +223,11 @@ export class AiWorker extends BaseWorker { deliveryOptions: { maxAttempts: 3, timeoutMs: 10_000, + ...(pending.deliveryHeaders ? { headers: pending.deliveryHeaders } : {}), }, fallbackChain: pending.fallbackChain, + campaignId: pending.campaignId, + workflowInstanceId: pending.workflowInstanceId, }; const envelope = buildStreamEvent( @@ -230,12 +242,15 @@ export class AiWorker extends BaseWorker { const scheduledAt = pending.scheduledAt ? new Date(pending.scheduledAt).getTime() : now; if (scheduledAt > now) { - await this.db.insert(scheduledPayloads).values({ - taskId, - // The send time rides in the payload so the scheduler can re-queue - // this from Postgres if Redis loses its data. - payload: { ...taskPayload, scheduledAt: pending.scheduledAt }, - }); + await this.db + .insert(scheduledPayloads) + .values({ + taskId, + // The send time rides in the payload so the scheduler can re-queue + // this from Postgres if Redis loses its data. + payload: { ...taskPayload, scheduledAt: pending.scheduledAt }, + }) + .onConflictDoNothing(); const scheduledEnvelope = buildStreamEvent( "notification.scheduled", @@ -305,7 +320,7 @@ export async function startAiWorker() { { stream: STREAMS.AI_PENDING, group: CONSUMER_GROUPS.AI, - consumer: `ai-${process.pid}`, + consumer: uniqueConsumerId("ai"), batchSize: config.WORKER_CONCURRENCY, }, )); diff --git a/src/services/api/handlers.ts b/src/services/api/handlers.ts index bf9cf06..a541bc5 100644 --- a/src/services/api/handlers.ts +++ b/src/services/api/handlers.ts @@ -43,7 +43,7 @@ import { readJsonBody, sendJson, sendNoContent, sendValidationError } from "./ht import type { RouteContext } from "./router.js"; import { globalEmitter, getPriorityBucket, normaliseTarget } from "@/shared/index.js"; import { metrics } from "@/metrics/index.js"; -import { eq, desc, and, lt, lte, gte, sql, like, or, isNotNull, inArray } from "drizzle-orm"; +import { eq, desc, and, lte, gte, sql, like, or, isNotNull, inArray } from "drizzle-orm"; import { messageLogs, workflowDefinitions, @@ -177,6 +177,106 @@ async function persistUsers(deps: Deps, users: InlineUserLike[], projectId: stri }); } +/** + * A dead-letter entry, read the way StreamConsumer writes it: the original + * event JSON under `data`, carrying a `dlq` block that records which stream it + * failed on. Flat `payload`/`eventType` fields are accepted for entries written + * by hand or by older versions. + */ +interface DlqEntry { + id: string; + eventType: string; + payload: any; + projectId?: string; + error: string; + timestamp: string; + /** The event as it was published, without its `dlq` block. */ + event: Record | null; + originalStream?: string; + fields: Record; +} + +function parseDlqEntry(id: string, rawFields: string[]): DlqEntry { + const fields: Record = {}; + for (let i = 0; i < rawFields.length; i += 2) { + fields[rawFields[i]!] = rawFields[i + 1]!; + } + + let parsed: Record | null = null; + if (fields.data) { + try { + parsed = JSON.parse(fields.data); + } catch { + parsed = null; + } + } + const { dlq, ...event } = parsed ?? {}; + + let payload: any = fields; + if (parsed) { + payload = event.payload; + } else if (fields.payload) { + try { + payload = JSON.parse(fields.payload); + } catch { + payload = fields.payload; + } + } + + const ms = Number(id.split("-")[0]); + return { + id, + eventType: event.type ?? fields.eventType ?? fields.event_type ?? "unknown", + payload, + projectId: payload?.projectId ?? payload?.project_id ?? fields.projectId, + error: dlq?.reason ?? fields.error ?? fields.reason ?? "Dead letter payload", + timestamp: + dlq?.ackedAt ?? + fields.timestamp ?? + (Number.isFinite(ms) ? new Date(ms).toISOString() : new Date().toISOString()), + event: parsed ? event : null, + originalStream: dlq?.originalStream, + fields, + }; +} + +/** + * Ids that live in uuid columns. Anything else can match no row, and passing it + * to Postgres fails the query outright — a 500 for what is simply not found. + */ +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const isUuid = (v: string | undefined): v is string => typeof v === "string" && UUID_RE.test(v); + +/** Opaque position in the delivery log: a row's full-precision timestamp and id. */ +function encodeLogCursor(ts: string, id: string): string { + return Buffer.from(JSON.stringify([ts, id])).toString("base64url"); +} + +function decodeLogCursor(cursor: string | null | undefined): { ts: string; id: string } | null { + if (!cursor) return null; + // Cursors handed out before this format were a millisecond timestamp. + if (/^d+$/.test(cursor)) { + return { + ts: new Date(Number(cursor)).toISOString(), + id: "ffffffff-ffff-ffff-ffff-ffffffffffff", + }; + } + try { + const [ts, id] = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); + if ( + typeof ts === "string" && + typeof id === "string" && + !Number.isNaN(Date.parse(ts)) && + /^[0-9a-f-]{36}$/i.test(id) + ) { + return { ts, id }; + } + } catch { + // fall through + } + return null; +} + function getQueryParam(ctx: RouteContext, key: string): string | undefined { if (!ctx.query) return undefined; if (typeof ctx.query.get === "function") { @@ -662,34 +762,82 @@ export function createHandlers(deps: Deps) { ); } - if (cursor) { - const cursorDate = new Date(parseInt(cursor, 10)); - conditions.push(lt(messageLogs.timestamp, cursorDate)); - } + // One entry per task, newest activity first. Rows are scanned in + // (timestamp, id) order — a batch of deliveries shares one timestamp, so + // the timestamp alone cannot mark a place — and a task is listed at its + // newest row. The cursor is that row for the first task of the next page, + // and any task with a row newer than the cursor was listed on an earlier + // page. Paging on a millisecond timestamp over `limit * 2` rows skipped + // tasks that shared a timestamp and dropped any that did not fit the slice. + const start = decodeLogCursor(cursor); + const tsText = sql`${messageLogs.timestamp}::text`; + const position = (ts: string, id: string) => sql`(${ts}::timestamptz, ${id}::uuid)`; + const rowKey = sql`(${messageLogs.timestamp}, ${messageLogs.id})`; - const logs = await deps.db - .select() - .from(messageLogs) - .where(and(...conditions)) - .orderBy(desc(messageLogs.timestamp)) - .limit(limit * 2); + const order: string[] = []; + const rowsByTask = new Map(); + let nextCursor: string | null = null; + let scanFrom = start; + let inclusive = true; + const batchSize = Math.max(limit * 4, 50); - // Deduplicate by taskId preserving the latest status per taskId - const deduplicatedMap = new Map(); - for (const log of logs) { - if (!deduplicatedMap.has(log.taskId)) { - deduplicatedMap.set(log.taskId, log); - } else { - const existing = deduplicatedMap.get(log.taskId)!; - if (existing.status === "dispatched" && log.status !== "dispatched") { - deduplicatedMap.set(log.taskId, log); - } + for (let round = 0; round < 50 && nextCursor === null; round++) { + const batch = await deps.db + .select({ row: messageLogs, ts: tsText }) + .from(messageLogs) + .where( + and( + ...conditions, + scanFrom + ? sql`${rowKey} ${inclusive ? sql`<=` : sql`<`} ${position(scanFrom.ts, scanFrom.id)}` + : undefined, + ), + ) + .orderBy(desc(messageLogs.timestamp), desc(messageLogs.id)) + .limit(batchSize); + + const fresh = [...new Set(batch.map((b) => b.row.taskId))].filter((t) => !rowsByTask.has(t)); + const listedEarlier = new Set(); + if (start && fresh.length > 0) { + const newer = await deps.db + .selectDistinct({ taskId: messageLogs.taskId }) + .from(messageLogs) + .where( + and( + ...conditions, + inArray(messageLogs.taskId, fresh), + sql`${rowKey} > ${position(start.ts, start.id)}`, + ), + ); + for (const r of newer) listedEarlier.add(r.taskId); } - } - const deduplicatedLogs = Array.from(deduplicatedMap.values()).slice(0, limit); - const nextCursor = - logs.length === limit * 2 ? logs[logs.length - 1]!.timestamp.getTime().toString() : null; + for (const { row, ts } of batch) { + if (listedEarlier.has(row.taskId)) continue; + let rows = rowsByTask.get(row.taskId); + if (!rows) { + if (order.length === limit) { + nextCursor = encodeLogCursor(ts, row.id); + break; + } + rows = []; + order.push(row.taskId); + rowsByTask.set(row.taskId, rows); + } + rows.push(row); + } + + if (batch.length < batchSize) break; + const last = batch[batch.length - 1]!; + scanFrom = { ts: last.ts, id: last.row.id }; + inclusive = false; + } + + // The latest outcome, rather than the dispatch that preceded it. + const deduplicatedLogs = order.map((taskId) => { + const rows = rowsByTask.get(taskId)!; + return rows.find((r) => r.status !== "dispatched") ?? rows[0]!; + }); sendJson(res, 200, { logs: deduplicatedLogs, nextCursor }); } @@ -875,7 +1023,9 @@ export function createHandlers(deps: Deps) { ctx: RouteContext, ): Promise { const instanceId = ctx.params.id!; - const workflow = await deps.workflowRepo.getInstance(ctx.projectId!, instanceId); + const workflow = isUuid(instanceId) + ? await deps.workflowRepo.getInstance(ctx.projectId!, instanceId) + : null; if (!workflow) return sendJson(res, 404, { error: "workflow_not_found", id: instanceId }); sendJson(res, 200, workflow); @@ -888,7 +1038,8 @@ export function createHandlers(deps: Deps) { ctx: RouteContext, ): Promise { const instanceId = ctx.params.id!; - const canceled = await deps.workflowRepo.cancelInstance(ctx.projectId!, instanceId); + const canceled = + isUuid(instanceId) && (await deps.workflowRepo.cancelInstance(ctx.projectId!, instanceId)); if (!canceled) return sendJson(res, 400, { error: "workflow_not_cancelable", id: instanceId }); logger.info({ instanceId }, "workflow canceled"); @@ -956,7 +1107,7 @@ export function createHandlers(deps: Deps) { ctx: RouteContext, ): Promise { const id = ctx.params.id!; - const deleted = await deps.projectRepo.delete(id); + const deleted = isUuid(id) && (await deps.projectRepo.delete(id)); if (!deleted) return sendJson(res, 404, { error: "project_not_found" }); sendNoContent(res); @@ -979,7 +1130,7 @@ export function createHandlers(deps: Deps) { if (!parsed.success) return sendValidationError(res, parsed.error); - const updated = await deps.projectRepo.updateSettings(id, parsed.data); + const updated = isUuid(id) && (await deps.projectRepo.updateSettings(id, parsed.data)); if (!updated) return sendJson(res, 404, { error: "project_not_found" }); // Drop the engine's cached throttle overrides so the change applies to the @@ -1009,6 +1160,11 @@ export function createHandlers(deps: Deps) { .safeParse(await readJsonBody(req)); const role = parsed.success ? parsed.data.role : "admin"; + // A key for a project that does not exist would fail the foreign key. + if (!isUuid(id) || !(await deps.projectRepo.findThrottleSettings(id))) { + return sendJson(res, 404, { error: "project_not_found" }); + } + const apiKey = `nk_live_${randomBytes(32).toString("hex")}`; const apiKeyHash = createHash("sha256").update(apiKey).digest("hex"); @@ -1023,6 +1179,7 @@ export function createHandlers(deps: Deps) { ctx: RouteContext, ): Promise { const id = ctx.params.id!; + if (!isUuid(id)) return sendJson(res, 404, { error: "project_not_found" }); const keys = await deps.projectRepo.listApiKeys(id); sendJson(res, 200, { keys }); } @@ -1034,7 +1191,7 @@ export function createHandlers(deps: Deps) { ctx: RouteContext, ): Promise { const { id, keyId } = ctx.params; - const deleted = await deps.projectRepo.deleteApiKey(id!, keyId!); + const deleted = isUuid(id) && isUuid(keyId) && (await deps.projectRepo.deleteApiKey(id, keyId)); if (!deleted) return sendJson(res, 404, { error: "key_not_found" }); await deps.redis.native.publish("apikey.invalidated", "*"); @@ -1234,41 +1391,40 @@ export function createHandlers(deps: Deps) { "COUNT", "100", ); - const allMessages = (rawEntries || []).map(([id, fields]: [string, string[]]) => { - const fieldMap: Record = {}; - for (let i = 0; i < fields.length; i += 2) { - fieldMap[fields[i]!] = fields[i + 1]!; - } - let payload: any = fieldMap; - if (fieldMap.payload) { - try { - payload = JSON.parse(fieldMap.payload); - } catch { - payload = fieldMap.payload; - } - } - return { - id, - eventType: fieldMap.eventType || fieldMap.event_type || "unknown", - payload, - error: fieldMap.error || fieldMap.reason || "Dead letter payload", - timestamp: fieldMap.timestamp || new Date().toISOString(), - }; - }); + const allMessages = (rawEntries || []).map(([id, fields]: [string, string[]]) => + parseDlqEntry(id, fields), + ); const messages = ctx.projectId - ? allMessages.filter((m) => { - const pId = m.payload?.projectId ?? m.payload?.project_id ?? (m as any).projectId; - return pId === ctx.projectId; - }) + ? allMessages.filter((m) => m.projectId === ctx.projectId) : allMessages; - sendJson(res, 200, { messages: messages.slice(0, 50) }); + sendJson(res, 200, { + messages: messages.slice(0, 50).map((m) => ({ + id: m.id, + eventType: m.eventType, + payload: m.payload, + error: m.error, + timestamp: m.timestamp, + originalStream: m.originalStream, + })), + }); } catch { sendJson(res, 200, { messages: [] }); } } + /** The entry, or null when it does not exist or belongs to another project. */ + async function findDlqEntry(ctx: RouteContext, messageId: string): Promise { + // Redis rejects a malformed stream id rather than matching nothing. + if (!/^\d+-\d+$/.test(messageId)) return null; + const rawEntries = await deps.redis.native.xrange(STREAMS.DEAD_LETTER, messageId, messageId); + if (!rawEntries || rawEntries.length === 0 || !rawEntries[0]) return null; + const entry = parseDlqEntry(rawEntries[0][0], rawEntries[0][1]); + if (ctx.projectId && entry.projectId !== ctx.projectId) return null; + return entry; + } + // ── POST /v1/dlq/replay — replayDLQMessage ──────────────────────────────── async function replayDLQMessage( req: IncomingMessage, @@ -1284,46 +1440,35 @@ export function createHandlers(deps: Deps) { } try { - const rawEntries = await deps.redis.native.xrange(STREAMS.DEAD_LETTER, messageId, messageId); - if (!rawEntries || rawEntries.length === 0 || !rawEntries[0]) { + const entry = await findDlqEntry(ctx, messageId); + if (!entry) { sendJson(res, 404, { error: "dlq_message_not_found" }); return; } - const fields = rawEntries[0][1]; - const fieldMap: Record = {}; - for (let i = 0; i < fields.length; i += 2) { - fieldMap[fields[i]!] = fields[i + 1]!; - } - - let payload: any = null; - if (fieldMap.payload) { - try { - payload = JSON.parse(fieldMap.payload); - } catch {} - } - - if (ctx.projectId) { - const pId = payload?.projectId ?? payload?.project_id ?? (fieldMap as any).projectId; - if (pId !== ctx.projectId) { - sendJson(res, 404, { error: "dlq_message_not_found" }); - return; - } - } - - const priority = fieldMap.priority || "normal"; - const p = getPriorityBucket(priority); + // Back onto the stream it failed on: a failed send belongs with delivery, + // not at the start of the pipeline, where its payload does not parse. + const replayable = new Set(Object.values(STREAMS)); + replayable.delete(STREAMS.DEAD_LETTER); + const inbound = + STREAMS[ + `INBOUND_${getPriorityBucket(entry.payload?.priority ?? entry.fields.priority).toUpperCase()}` as keyof typeof STREAMS + ] ?? STREAMS.INBOUND_NORMAL; const targetStream = - STREAMS[`INBOUND_${p.toUpperCase()}` as keyof typeof STREAMS] || STREAMS.INBOUND_NORMAL; + entry.originalStream && replayable.has(entry.originalStream) + ? entry.originalStream + : inbound; const xaddArgs: string[] = []; - for (const [k, v] of Object.entries(fieldMap)) { - xaddArgs.push(k, v); + if (entry.event) { + xaddArgs.push("data", JSON.stringify(entry.event)); + } else { + for (const [k, v] of Object.entries(entry.fields)) xaddArgs.push(k, v); } await deps.redis.native.xadd(targetStream, "*", ...xaddArgs); await deps.redis.native.xdel(STREAMS.DEAD_LETTER, messageId); - sendJson(res, 200, { success: true, replayedId: messageId }); + sendJson(res, 200, { success: true, replayedId: messageId, stream: targetStream }); } catch (err: any) { sendJson(res, 500, { error: "replay_failed", message: err.message }); } @@ -1339,34 +1484,10 @@ export function createHandlers(deps: Deps) { if (!messageId) return sendJson(res, 400, { error: "missing_id" }); try { - if (ctx.projectId) { - const rawEntries = await deps.redis.native.xrange( - STREAMS.DEAD_LETTER, - messageId, - messageId, - ); - if (!rawEntries || rawEntries.length === 0 || !rawEntries[0]) { - sendJson(res, 404, { error: "dlq_message_not_found" }); - return; - } - const fields = rawEntries[0][1]; - const fieldMap: Record = {}; - for (let i = 0; i < fields.length; i += 2) { - fieldMap[fields[i]!] = fields[i + 1]!; - } - let payload: any = null; - if (fieldMap.payload) { - try { - payload = JSON.parse(fieldMap.payload); - } catch {} - } - const pId = payload?.projectId ?? payload?.project_id ?? (fieldMap as any).projectId; - if (pId !== ctx.projectId) { - sendJson(res, 404, { error: "dlq_message_not_found" }); - return; - } + if (!(await findDlqEntry(ctx, messageId))) { + sendJson(res, 404, { error: "dlq_message_not_found" }); + return; } - await deps.redis.native.xdel(STREAMS.DEAD_LETTER, messageId); sendJson(res, 200, { success: true }); } catch (err: any) { @@ -1766,10 +1887,13 @@ code{background:#f4f4f5;padding:.1rem .35rem;border-radius:4px} if (ENGAGEMENT.includes(row.kind as Engagement)) { bucket[row.kind as Engagement] += n; - } else { - // A delivery row. `sent` is every task that reached a provider at all, - // which is the denominator a bounce rate is measured against. + } else if (row.kind === "dispatched") { + // `sent` is every task that reached a provider at all, which is the + // denominator a bounce rate is measured against. Each attempt logs a + // dispatched row besides its outcome row, and the two are grouped + // separately, so counting both counted every message twice. bucket.sent += n; + } else { if (row.status === "delivered") bucket.delivered += n; if (row.status === "failed") bucket.failed += n; } diff --git a/src/services/delivery/main.ts b/src/services/delivery/main.ts index edcb78a..62d1ac7 100644 --- a/src/services/delivery/main.ts +++ b/src/services/delivery/main.ts @@ -15,6 +15,7 @@ import { registry, buildStreamEvent, type NotificationDispatchedPayload, + type DispatchedTaskPayload, type RenderedContent, PUBSUB_CHANNELS, } from "@/index.js"; @@ -73,6 +74,37 @@ let subscriber: Redis | null = null; /** How long a send may run before its dispatched log entry is published alone. */ const DISPATCH_LOG_DELAY_MS = 1_000; +/** + * Wait before re-trying a send that failed at the provider, by prior failures. + * The first step matches the circuit breaker's reset timeout, so a retry of a + * task the open breaker turned away reaches the provider's recovery probe + * rather than the same closed door. + */ +const RETRY_BACKOFF_MS = [30_000, 120_000, 600_000]; + +/** + * Names the provider at `transports[i]` for its rate limit and circuit breaker. + * The rate limit's budget lives in Redis and is shared by every delivery + * process, so the name must come out the same everywhere: the class name, + * numbered when a channel has two providers of one class. + */ +function providerKey(channel: string, transports: any[], i: number): string { + const name = transports[i].constructor?.name ?? "provider"; + const nth = transports.slice(0, i).filter((t) => t.constructor?.name === name).length; + return nth === 0 ? `${channel}:${name}` : `${channel}:${name}#${nth + 1}`; +} + +/** A provider's refusal, keeping whether it said another attempt could work. */ +function providerFailure(res: { error?: string; retryable?: boolean }): Error { + const err = new Error(res.error ?? "Transport failed"); + (err as any).retryable = res.retryable; + return err; +} + +export function retryDelayMs(priorFailures: number): number { + return RETRY_BACKOFF_MS[Math.min(priorFailures, RETRY_BACKOFF_MS.length - 1)]!; +} + export interface DeliveryWorkerOptions extends WorkerOptions { transportRegistry: any; idempotency: any; @@ -196,6 +228,51 @@ export class DeliveryWorker extends BaseWorker { return renderWithTemplate(template, task.templateVariables); } + /** + * Parks a task in Postgres and the scheduler until `sendAt`. The payload + * carries the send time, so the scheduler can rebuild it if Redis loses its + * data. + */ + private async reschedule( + task: DispatchedTaskPayload, + sendAt: Date, + reason: string, + ): Promise { + const scheduledAt = sendAt.toISOString(); + await this.db + .insert(scheduledPayloads) + .values({ taskId: task.taskId, payload: { ...task, scheduledAt } }) + .onConflictDoUpdate({ + target: scheduledPayloads.taskId, + set: { payload: drizzleSql`EXCLUDED.payload` }, + }); + + await this.scheduledProducer.publish( + buildStreamEvent( + "notification.scheduled", + { + projectId: task.projectId, + enrichedEventId: task.enrichedEventId, + taskId: task.taskId, + scheduledAt, + }, + "delivery", + `${task.taskId}:${reason}:${Date.now()}`, + ), + ); + } + + /** Takes a slot from the provider's rate limit, if it has one. */ + private async providerGate( + channel: string, + transports: any[], + i: number, + ): Promise<{ allowed: boolean; retryAfterMs: number }> { + const limits = transports[i].limits; + if (!limits) return { allowed: true, retryAfterMs: 0 }; + return this.providerThrottle.check(providerKey(channel, transports, i), limits); + } + private getBreaker(name: string): CircuitBreaker { let breaker = this.breakers.get(name); if (!breaker) { @@ -219,6 +296,11 @@ export class DeliveryWorker extends BaseWorker { } const task = payloadResult.data as NotificationDispatchedPayload; + // Attempts made on earlier trips through the scheduler, plus this one. + const attemptNo = (task.deliveryAttemptCount ?? 0) + attempt; + // What gets parked for a retry: the task as it arrived, before rendering + // and the transport's abort signal were attached to it. + const original: DispatchedTaskPayload = { ...task }; const fallbackToNextChannel = async (reason: string) => { if (task.fallbackChain && task.fallbackChain.length > 0 && task.recipient) { @@ -238,6 +320,9 @@ export class DeliveryWorker extends BaseWorker { recipient: task.recipient, scheduledAt: undefined, fallbackChain: remainingChain.length > 0 ? remainingChain : undefined, + campaignId: task.campaignId, + workflowInstanceId: task.workflowInstanceId, + throttleCounted: true, }; const p = getPriorityBucket(task.priority); @@ -288,10 +373,9 @@ export class DeliveryWorker extends BaseWorker { failureReason: "no transport registered for channel", failureCode: "no_transport", retryable: false, - attempt, + attempt: attemptNo, templateId: task.templateId, - workflowInstanceId: - event.metadata.source === "workflow" ? event.metadata.traceId : undefined, + workflowInstanceId: task.workflowInstanceId, campaignId: task.campaignId, }, "delivery", @@ -302,87 +386,67 @@ export class DeliveryWorker extends BaseWorker { return; } - const limitConfig = transports[0].limits; - if (limitConfig) { - const { allowed, retryAfterMs } = await this.providerThrottle.check( - task.channel, - limitConfig, - ); - if (!allowed) { - task.throttleAttemptCount = (task.throttleAttemptCount ?? 0) + 1; - const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3; + // Each provider has its own budget: start at the first with room left. The + // channel is out of capacity only once every provider for it is. + let startAt = -1; + let retryAfterMs = Infinity; + for (let i = 0; i < transports.length; i++) { + const gate = await this.providerGate(task.channel, transports, i); + if (gate.allowed) { + startAt = i; + break; + } + retryAfterMs = Math.min(retryAfterMs, gate.retryAfterMs); + } + if (startAt === -1) { + task.throttleAttemptCount = (task.throttleAttemptCount ?? 0) + 1; + const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3; - if (task.throttleAttemptCount > maxAttempts) { - this.logger.warn( - { messageId: message.id, taskId: task.taskId, attempts: task.throttleAttemptCount }, - "provider rate limit max attempts exceeded", - ); - const fallbackTriggered = await fallbackToNextChannel("provider_throttle_exceeded"); - if (!fallbackTriggered) { - this.globalEmitter.emit( - "delivery:failed", - task.taskId, - "provider throttle exceeded", - task.channel, - task.projectId, - ); - await this.eventProcessor.add( - buildStreamEvent( - "notification.failed", - { - projectId: task.projectId, - taskId: task.taskId, - enrichedEventId: task.enrichedEventId, - recipientId: task.recipientId, - channel: task.channel, - failureReason: "provider rate limit max attempts exceeded", - failureCode: "provider_throttle_exceeded", - retryable: false, - attempt, - templateId: task.templateId, - workflowInstanceId: - event.metadata.source === "workflow" ? event.metadata.traceId : undefined, - campaignId: task.campaignId, - }, - "delivery", - event.metadata.traceId, - ), - ); - } - return; - } - - // The retry time rides in the payload so the scheduler can re-queue - // this from Postgres if Redis loses its data. - const retryAt = new Date(Date.now() + retryAfterMs).toISOString(); - await this.db - .insert(scheduledPayloads) - .values({ - taskId: task.taskId, - payload: { ...task, scheduledAt: retryAt }, - }) - .onConflictDoUpdate({ - target: scheduledPayloads.taskId, - set: { payload: drizzleSql`EXCLUDED.payload` }, - }); - - await this.scheduledProducer.publish( - buildStreamEvent( - "notification.scheduled", - { - projectId: task.projectId, - enrichedEventId: task.enrichedEventId, - taskId: task.taskId, - scheduledAt: retryAt, - throttleAttemptCount: task.throttleAttemptCount, - }, - "delivery", - `${task.taskId}:throttle:${Date.now()}`, - ), + if (task.throttleAttemptCount > maxAttempts) { + this.logger.warn( + { messageId: message.id, taskId: task.taskId, attempts: task.throttleAttemptCount }, + "provider rate limit max attempts exceeded", ); - + const fallbackTriggered = await fallbackToNextChannel("provider_throttle_exceeded"); + if (!fallbackTriggered) { + this.globalEmitter.emit( + "delivery:failed", + task.taskId, + "provider throttle exceeded", + task.channel, + task.projectId, + ); + await this.eventProcessor.add( + buildStreamEvent( + "notification.failed", + { + projectId: task.projectId, + taskId: task.taskId, + enrichedEventId: task.enrichedEventId, + recipientId: task.recipientId, + channel: task.channel, + failureReason: "provider rate limit max attempts exceeded", + failureCode: "provider_throttle_exceeded", + retryable: false, + attempt: attemptNo, + templateId: task.templateId, + workflowInstanceId: task.workflowInstanceId, + campaignId: task.campaignId, + }, + "delivery", + event.metadata.traceId, + ), + ); + } return; } + + await this.reschedule( + { ...original, throttleAttemptCount: task.throttleAttemptCount }, + new Date(Date.now() + retryAfterMs), + "throttle", + ); + return; } const idempotencyKey = task.taskId; @@ -433,9 +497,8 @@ export class DeliveryWorker extends BaseWorker { recipientId: task.recipientId, channel: task.channel, templateId: task.templateId, - attempt, - workflowInstanceId: - event.metadata.source === "workflow" ? event.metadata.traceId : undefined, + attempt: attemptNo, + workflowInstanceId: task.workflowInstanceId, campaignId: task.campaignId, }, "delivery", @@ -450,7 +513,40 @@ export class DeliveryWorker extends BaseWorker { clearTimeout(dispatchLogTimer); if (dispatchLogged) return payload; dispatchLogged = true; - return { ...payload, attempt, dispatchedAt }; + return { ...payload, attempt: attemptNo, dispatchedAt }; + }; + + let retryScheduled = false; + /** + * Re-queues a send the provider turned away, unless it said the failure is + * permanent or the task is out of attempts. An outage, a timeout or an open + * circuit is exactly what a later attempt can get past; failing the task + * outright turned every blip into lost mail. The attempt stays in the log as + * dispatched, and the task's outcome is whatever its last attempt comes to. + */ + const retryLater = async (failure: { retryable?: boolean; error?: string }) => { + const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3; + if (failure.retryable === false || attemptNo >= maxAttempts) return false; + const delayMs = retryDelayMs(task.deliveryAttemptCount ?? 0); + await this.reschedule( + { ...original, deliveryAttemptCount: attemptNo }, + new Date(Date.now() + delayMs), + "retry", + ); + retryScheduled = true; + metrics.deliveryFailed.inc({ channel: task.channel, reason: "retry_scheduled" }); + this.logger.warn( + { + taskId: task.taskId, + channel: task.channel, + attempt: attemptNo, + maxAttempts, + delayMs, + error: failure.error, + }, + "provider send failed — retrying later", + ); + return true; }; try { @@ -469,7 +565,13 @@ export class DeliveryWorker extends BaseWorker { if (task.channel === "push") { let lastResult: any = { success: false, error: "No transports" }; - for (const transport of transports) { + for (let i = startAt; i < transports.length; i++) { + const transport = transports[i]; + // The starting provider's slot was taken above. Failing over to a + // later one spends that provider's own budget, or skips it if spent. + if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) { + continue; + } const elapsedMs = Date.now() - tDispatchStart; const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs); if (remainingBudgetMs <= 0) { @@ -484,7 +586,7 @@ export class DeliveryWorker extends BaseWorker { try { const tProv = Date.now(); - const breaker = this.getBreaker(`${task.channel}:${transport.constructor.name}`); + const breaker = this.getBreaker(providerKey(task.channel, transports, i)); lastResult = await breaker.execute(async () => { const controller = new AbortController(); @@ -504,7 +606,7 @@ export class DeliveryWorker extends BaseWorker { }), ]); if (!res.success && !res.invalidToken) { - throw new Error(res.error ?? "Transport failed"); + throw providerFailure(res); } return res; } finally { @@ -515,7 +617,7 @@ export class DeliveryWorker extends BaseWorker { (global as any)._telemetry.provider += Date.now() - tProv; if (lastResult.success || lastResult.invalidToken) break; } catch (err: any) { - lastResult = { success: false, error: err.message }; + lastResult = { success: false, error: err.message, retryable: err.retryable }; } } @@ -560,10 +662,9 @@ export class DeliveryWorker extends BaseWorker { failureReason: "invalid_token", failureCode: "push_failure", retryable: false, - attempt, + attempt: attemptNo, templateId: task.templateId, - workflowInstanceId: - event.metadata.source === "workflow" ? event.metadata.traceId : undefined, + workflowInstanceId: task.workflowInstanceId, campaignId: task.campaignId, }), "delivery", @@ -614,8 +715,7 @@ export class DeliveryWorker extends BaseWorker { deliveredAt: new Date().toISOString(), providerMessageId, templateId: task.templateId, - workflowInstanceId: - event.metadata.source === "workflow" ? event.metadata.traceId : undefined, + workflowInstanceId: task.workflowInstanceId, campaignId: task.campaignId, }), "delivery", @@ -623,7 +723,7 @@ export class DeliveryWorker extends BaseWorker { ), ), ); - } else { + } else if (!(await retryLater(lastResult))) { this.logger.warn( { taskId: task.taskId, error: lastResult.error }, "push delivery failed", @@ -659,10 +759,9 @@ export class DeliveryWorker extends BaseWorker { failureReason: lastResult.error ?? "push delivery failed", failureCode: "push_failure", retryable: false, - attempt, + attempt: attemptNo, templateId: task.templateId, - workflowInstanceId: - event.metadata.source === "workflow" ? event.metadata.traceId : undefined, + workflowInstanceId: task.workflowInstanceId, campaignId: task.campaignId, }), "delivery", @@ -679,7 +778,13 @@ export class DeliveryWorker extends BaseWorker { } else { // Other channels (email, sms, webhook, whatsapp) use the pre-resolved destination let result: any = { success: false, error: "No transports" }; - for (const transport of transports) { + for (let i = startAt; i < transports.length; i++) { + const transport = transports[i]; + // The starting provider's slot was taken above. Failing over to a + // later one spends that provider's own budget, or skips it if spent. + if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) { + continue; + } const elapsedMs = Date.now() - tDispatchStart; const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs); if (remainingBudgetMs <= 0) { @@ -694,7 +799,7 @@ export class DeliveryWorker extends BaseWorker { try { const tProv = Date.now(); - const breaker = this.getBreaker(`${task.channel}:${transport.constructor.name}`); + const breaker = this.getBreaker(providerKey(task.channel, transports, i)); result = await breaker.execute(async () => { const controller = new AbortController(); @@ -714,7 +819,7 @@ export class DeliveryWorker extends BaseWorker { }), ]); if (!res.success) { - throw new Error(res.error ?? "Transport failed"); + throw providerFailure(res); } return res; } finally { @@ -725,7 +830,7 @@ export class DeliveryWorker extends BaseWorker { (global as any)._telemetry.provider += Date.now() - tProv; if (result.success) break; } catch (err: any) { - result = { success: false, error: err.message }; + result = { success: false, error: err.message, retryable: err.retryable }; } } @@ -763,8 +868,7 @@ export class DeliveryWorker extends BaseWorker { deliveredAt: new Date().toISOString(), providerMessageId, templateId: task.templateId, - workflowInstanceId: - event.metadata.source === "workflow" ? event.metadata.traceId : undefined, + workflowInstanceId: task.workflowInstanceId, campaignId: task.campaignId, }), "delivery", @@ -772,7 +876,7 @@ export class DeliveryWorker extends BaseWorker { ), ), ); - } else { + } else if (!(await retryLater(result))) { publishPromises.push( this.outboxUpdateProcessor.add({ taskId: task.taskId, @@ -794,10 +898,9 @@ export class DeliveryWorker extends BaseWorker { failureReason: result.error ?? "delivery failed completely", failureCode: "provider_error", retryable: false, - attempt, + attempt: attemptNo, templateId: task.templateId, - workflowInstanceId: - event.metadata.source === "workflow" ? event.metadata.traceId : undefined, + workflowInstanceId: task.workflowInstanceId, campaignId: task.campaignId, }), "delivery", @@ -807,7 +910,7 @@ export class DeliveryWorker extends BaseWorker { ); } - if (!result.success) { + if (!result.success && !retryScheduled) { this.logger.warn( { taskId: task.taskId, channel: task.channel, error: result.error }, "delivery failed completely across providers", @@ -825,7 +928,7 @@ export class DeliveryWorker extends BaseWorker { if (!fallbackTriggered) { throw new NonRetryableError(result.error ?? "delivery failed"); } - } else { + } else if (result.success) { this.logger.debug( { taskId: task.taskId, channel: task.channel, messageId: result.providerMessageId }, "notification delivered", @@ -845,9 +948,14 @@ export class DeliveryWorker extends BaseWorker { }); const flushTime = Date.now() - tFlushStart; - void this.idempotency.markProcessed(idempotencyKey).catch((err: any) => { - this.logger.warn({ err, taskId: task.taskId }, "failed to upgrade idempotency TTL"); - }); + if (retryScheduled) { + // Released so the re-queued attempt can take the lease. + await this.idempotency.unmark(idempotencyKey).catch(() => {}); + } else { + void this.idempotency.markProcessed(idempotencyKey).catch((err: any) => { + this.logger.warn({ err, taskId: task.taskId }, "failed to upgrade idempotency TTL"); + }); + } const t = ((global as any)._telemetry = (global as any)._telemetry || { count: 0, diff --git a/src/services/engine/main.ts b/src/services/engine/main.ts index b8b7c9c..44f9b1f 100644 --- a/src/services/engine/main.ts +++ b/src/services/engine/main.ts @@ -414,7 +414,13 @@ export class EngineWorker extends BaseWorker { // Quiet hours check const qh = enriched.recipient.preferences.quietHours; if (qh && qh.length > 0 && enriched.priority !== "critical") { - const qhResult = isInQuietHours(enriched.recipient.timezone, qh); + // Judged at the moment the message would go out, not the moment the + // engine sees it: a send booked for next week must not be pulled + // forward to tonight's quiet-hours end, and one booked for 3am must + // wait until the quiet hours are over. + const requestedAt = enriched.scheduledAt ? new Date(enriched.scheduledAt) : new Date(); + const sendAt = requestedAt.getTime() > Date.now() ? requestedAt : new Date(); + const qhResult = isInQuietHours(enriched.recipient.timezone, qh, sendAt); if (qhResult.inQuietHours && qhResult.nextActiveTime) { this.logger.info( { @@ -444,18 +450,16 @@ export class EngineWorker extends BaseWorker { ); } - const throttleResult = await this.throttle.check( - enriched.projectId, - enriched.recipientId, - enriched.priority, - { - limit: projectThrottle.throttleLimit, - windowHours: projectThrottle.throttleWindowHours, - scheduledAt: enriched.scheduledAt, - // A replayed message must not be counted against the user twice. - messageId: idempotencyKey, - }, - ); + // A channel fallback was counted when the notification was first sent. + const throttleResult = enriched.throttleCounted + ? { allowed: true, count: 0, limit: 0 } + : await this.throttle.check(enriched.projectId, enriched.recipientId, enriched.priority, { + limit: projectThrottle.throttleLimit, + windowHours: projectThrottle.throttleWindowHours, + scheduledAt: enriched.scheduledAt, + // A replayed message must not be counted against the user twice. + messageId: idempotencyKey, + }); if (!throttleResult.allowed) { this.logger.info( { @@ -476,8 +480,7 @@ export class EngineWorker extends BaseWorker { return; } - // Gather AI Prompts - // Gather AI Prompts + // Template-level AI prompts, overridden by any on the request. let dbTemplate = null; if (enriched.templateId) { dbTemplate = await this.templateCache.getCachedTemplate( @@ -508,41 +511,10 @@ export class EngineWorker extends BaseWorker { // resulting opt-out applies to. const templateTopics: string[] = dbTemplate?.topics ?? []; - if (Object.keys(aiPrompts).length > 0) { - const aiPendingPayload = { - projectId: enriched.projectId, - enrichedEventId: event.id, - recipientId: enriched.recipientId, - channel: enriched.channel, - priority: enriched.priority, - templateId: enriched.templateId, - templateVariables: enriched.templateVariables, - recipient: enriched.recipient, - aiPrompts, - scheduledAt: enriched.scheduledAt, - fallbackChain: enriched.fallbackChain, - }; - - const aiPendingEnvelope = buildStreamEvent( - "notification.ai_pending", - aiPendingPayload as Record, - "engine", - event.metadata.traceId, - ); - - await this.aiPendingProducer.publish(aiPendingEnvelope); - this.logger.info( - { - messageId: message.id, - eventId: event.id, - recipientId: enriched.recipientId, - traceId: event.metadata.traceId, - }, - "task routed to AI worker", - ); - await idempotency.markProcessed(idempotencyKey, customTtl); - return; - } + // AI-generated messages take the same gates as any other — contacts, + // topic opt-outs and suppressions below — and are handed to the AI worker + // per contact once they have passed them. + const useAi = Object.keys(aiPrompts).length > 0; // Rendering happens in delivery, just before the send, so the rendered // body does not travel through the outbound stream. @@ -588,7 +560,8 @@ export class EngineWorker extends BaseWorker { }); for (const contact of activeContacts) { - if (contact.preferences?.optedOut) { + // This address opted out of one of the template's topics. + if (templateTopics.some((t) => contact.preferences?.topics?.[t] === false)) { continue; } @@ -670,8 +643,42 @@ export class EngineWorker extends BaseWorker { }, fallbackChain: enriched.fallbackChain, campaignId: enriched.campaignId, + workflowInstanceId: enriched.workflowInstanceId, }; + if (useAi) { + await this.aiPendingProducer.publish( + buildStreamEvent( + "notification.ai_pending", + { + projectId: enriched.projectId, + enrichedEventId: event.id, + recipientId: enriched.recipientId, + channel: enriched.channel, + priority: enriched.priority, + templateId: enriched.templateId, + templateVariables: enriched.templateVariables, + recipient: enriched.recipient, + aiPrompts, + scheduledAt: enriched.scheduledAt, + fallbackChain: enriched.fallbackChain, + taskId, + destination: resolvedDestination, + deliveryHeaders: unsubscribeHeaders, + campaignId: enriched.campaignId, + workflowInstanceId: enriched.workflowInstanceId, + }, + "engine", + event.metadata.traceId, + ), + ); + this.logger.debug( + { messageId: message.id, taskId, traceId: event.metadata.traceId }, + "task routed to AI worker", + ); + continue; + } + const envelope = buildStreamEvent( "notification.dispatched", taskPayload as Record, diff --git a/src/services/enricher/main.ts b/src/services/enricher/main.ts index c9e9842..cb4a150 100644 --- a/src/services/enricher/main.ts +++ b/src/services/enricher/main.ts @@ -11,6 +11,7 @@ import { buildStreamEvent, type NotificationCreatedPayload, type NotificationEnrichedPayload, + type NotificationChannel, type StreamEvent, } from "@/index.js"; import { IdempotencyGuard } from "@/index.js"; @@ -21,7 +22,7 @@ import { ContactRepository, } from "@/index.js"; import { TemplateCache } from "@/templates/index.js"; -import { getPriorityBucket, type WorkerOptions } from "@/shared/index.js"; +import { getPriorityBucket, globalEmitter, type WorkerOptions } from "@/shared/index.js"; import { startHealthReporter } from "@/workers/index.js"; import { createPriorityProducers, @@ -256,10 +257,13 @@ export class EnricherWorker extends BaseWorker { const profile = await this.userRepo.findRecordById(raw.projectId, raw.recipientId); if (!profile) return; - const prefs = await this.prefRepo.findByUserId(raw.projectId, raw.recipientId); - const optedOutTypes = new Set( - prefs.filter((p: any) => !p.optedIn).map((p: any) => p.eventType), - ); + // Opt-outs are keyed on the template's topics, as on the requested + // path. `event.type` is always "notification.created", which no user + // ever opts out of, so checking it here let every opt-out through. + const template = raw.templateId + ? await this.templateCache.getCachedTemplate(raw.projectId, raw.templateId) + : null; + const topics: string[] = template?.topics ?? []; const enrichedPayload: NotificationEnrichedPayload = { projectId: raw.projectId, @@ -275,8 +279,7 @@ export class EnricherWorker extends BaseWorker { locale: profile.language ?? "en", timezone: profile.timezone ?? "UTC", preferences: { - optedOut: - optedOutTypes.has(event.type) || profile.preferences.topics?.[event.type] === false, + optedOut: topics.some((t) => profile.preferences.topics?.[t] === false), channels: Object.entries(profile.preferences.channels ?? {}) .filter(([_, enabled]) => !enabled) .map(([channel]) => channel as any), @@ -311,6 +314,7 @@ export class EnricherWorker extends BaseWorker { "event enriched", ); } catch (err) { + await this.idempotency.unmark(dedupeId).catch(() => {}); throw err; } await Promise.all(publishPromises).catch(async (err) => { @@ -345,24 +349,16 @@ export class EnricherWorker extends BaseWorker { const maxUsers = readBaseConfig().SEGMENT_MAX_USERS; if (userIds.length > maxUsers) { + const reason = `Segment fan-out of ${userIds.length} exceeds limit of ${maxUsers}`; this.logger.error( { count: userIds.length, max: maxUsers, projectId: raw.projectId, eventId: event.id }, "Segment fan-out exceeds maximum allowed limit", ); - const p = getPriorityBucket(raw.priority ?? "normal"); - const producer = this.producers[p] ?? this.producers.normal; - await producer.publish( - buildStreamEvent( - "notification.failed", - { - projectId: raw.projectId, - rawEventId: event.id, - error: `Segment fan-out of ${userIds.length} exceeds limit of ${maxUsers}`, - }, - "enricher", - event.metadata.traceId, - ), - ); + // Reported to the caller, not published: a `notification.failed` on + // the enriched stream is not a payload the engine can read, so the + // refusal used to vanish there without a trace. + globalEmitter.emit("notification:failed", event.id, reason, event.type); + await this.idempotency.markProcessed(dedupeId); return; } @@ -374,15 +370,10 @@ export class EnricherWorker extends BaseWorker { : null; const topics: string[] = template?.topics ?? []; - const channels = - raw.channels && raw.channels.length > 0 ? raw.channels : (["email"] as any[]); + const channels: NotificationChannel[] = + raw.channels && raw.channels.length > 0 ? raw.channels : ["email"]; const isFallback = (raw as any).fallback === true; - // If fallback is true, we only emit the first channel, and pass the rest in fallbackChain. - // If fallback is false, we emit all channels concurrently. - const channelsToProcess = isFallback ? [channels[0]] : channels; - const fallbackChain = isFallback ? channels.slice(1) : undefined; - const chunkArray = (arr: T[], size: number) => Array.from({ length: Math.ceil(arr.length / size) }, (v, i) => arr.slice(i * size, i * size + size), @@ -410,9 +401,36 @@ export class EnricherWorker extends BaseWorker { }; for (const profile of profiles) { - for (const channel of channelsToProcess) { - const contacts = contactsByUser.get(profile.userId) ?? []; - const channelContacts = contacts.filter((contact: any) => contact.channel === channel); + const contacts = contactsByUser.get(profile.userId) ?? []; + // This user's addresses on a channel, less any that opted out of one + // of this template's topics. + const usable = (channel: string) => + contacts.filter( + (contact: any) => + contact.channel === channel && + !topics.some((t) => contact.preferences?.topics?.[t] === false), + ); + + // Without fallback every requested channel is sent at once. With it, + // start at the first channel this user can be reached on and chain the + // rest behind it — a user with no email address still gets the SMS. + let userChannels: NotificationChannel[] = channels; + let fallbackChain: NotificationChannel[] | undefined; + if (isFallback) { + const first = channels.findIndex((c: NotificationChannel) => usable(c).length > 0); + if (first === -1) { + this.logger.info( + { recipientId: profile.userId, channels }, + "no active contact on any fallback channel", + ); + continue; + } + userChannels = [channels[first]!]; + fallbackChain = channels.slice(first + 1); + } + + for (const channel of userChannels) { + const channelContacts = usable(channel); // Push resolves its active tokens at send time so token invalidation // remains current. Other channels need one task per address. const resolved: ({ id: string; target: string } | undefined)[] = @@ -430,7 +448,7 @@ export class EnricherWorker extends BaseWorker { projectId: raw.projectId, rawEventId: event.id, recipientId: profile.userId, - channel: channel, + channel, priority: "normal", templateId: raw.templateId, templateVariables: raw.data, @@ -464,6 +482,7 @@ export class EnricherWorker extends BaseWorker { contactId: contact?.id, destination, campaignId: raw.campaignId, + workflowInstanceId: raw.workflowInstanceId, }; const msgPriority = raw.priority ?? "normal"; @@ -520,6 +539,9 @@ export class EnricherWorker extends BaseWorker { "event enriched", ); } catch (err) { + // Release the marker, or a retry landing inside its TTL is taken for a + // duplicate and acked without ever being processed. + await this.idempotency.unmark(dedupeId).catch(() => {}); throw err; } diff --git a/src/services/events/main.ts b/src/services/events/main.ts index 034ab0e..738213d 100644 --- a/src/services/events/main.ts +++ b/src/services/events/main.ts @@ -14,7 +14,12 @@ import { workflowWaiters, workflowSteps, workflowInstances } from "@/db/schema.j import { eq, and, or, isNull, gt } from "drizzle-orm"; import { type WorkerOptions } from "@/shared/index.js"; import { startHealthReporter } from "@/workers/index.js"; -import { createStreamConsumer, createWorkerRuntime, shutdownWorker } from "@/workers/bootstrap.js"; +import { + createStreamConsumer, + createWorkerRuntime, + shutdownWorker, + uniqueConsumerId, +} from "@/workers/bootstrap.js"; loadEnv(); const config = readBaseConfig(); @@ -376,7 +381,7 @@ export async function startEventWorker() { { stream: STREAMS.EVENTS_INBOUND, group: CONSUMER_GROUPS.EVENTS, - consumer: `events-${process.pid}`, + consumer: uniqueConsumerId("events"), batchSize: eventsConcurrency, bufferAcks: true, // Log rows are not latency-sensitive; let reads fill up. diff --git a/src/services/scheduler/main.ts b/src/services/scheduler/main.ts index c186957..5d6489a 100644 --- a/src/services/scheduler/main.ts +++ b/src/services/scheduler/main.ts @@ -11,16 +11,18 @@ import { buildStreamEvent, type NotificationScheduledPayload, } from "@/index.js"; -import { getPriorityBucket, type WorkerOptions } from "@/shared/index.js"; +import { getPriorityBucket, globalEmitter, type WorkerOptions } from "@/shared/index.js"; import { scheduledPayloads } from "@/db/schema.js"; -import { asc, gt, inArray } from "drizzle-orm"; +import { and, asc, eq, gt, inArray, or, sql as drizzleSql } from "drizzle-orm"; import { startHealthReporter } from "@/workers/index.js"; import { createPriorityProducers, createStreamConsumer, createWorkerRuntime, shutdownWorker, + uniqueConsumerId, } from "@/workers/bootstrap.js"; +import { findBlockedAtRelease } from "./send-gate.js"; // ─── Bootstrap ───────────────────────────────────────────────────────────── loadEnv(); @@ -137,6 +139,11 @@ export async function executeSchedulerPoll( .where(inArray(scheduledPayloads.taskId, taskIds)); const payloadMap = new Map(dbPayloads.map((row: any) => [row.taskId, row.payload])); + // What the person did while this sat parked still applies. A failed + // lookup throws to the catch below: the claimed tasks reappear after the + // visibility timeout instead of going out unchecked. + const blocked = await findBlockedAtRelease(db, [...payloadMap.values()] as any[]); + const batchedEvents: Record<"critical" | "normal" | "low", Omit[]> = { critical: [], normal: [], @@ -170,6 +177,23 @@ export async function executeSchedulerPoll( } const dispatchPayload = payload as any; + + const blockReason = blocked.get(taskId); + if (blockReason) { + logger.info({ taskId, reason: blockReason }, "parked send no longer allowed — dropping"); + globalEmitter.emit("notification:skipped", { + projectId: dispatchPayload.projectId, + eventId: dispatchPayload.enrichedEventId, + recipientId: dispatchPayload.recipientId, + reason: blockReason, + }); + released.add(taskId); + dbCleanupIds.push(taskId); + const shard = parseInt(taskId.slice(-1), 16) || 0; + cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr); + continue; + } + const p = getPriorityBucket(dispatchPayload.priority); batchedEvents[p].push( @@ -194,7 +218,24 @@ export async function executeSchedulerPoll( if (dbCleanupIds.length > 0) { try { - await db.delete(scheduledPayloads).where(inArray(scheduledPayloads.taskId, dbCleanupIds)); + // Only the payload that was released. By now delivery may have + // throttled the task and parked it again under the same id with a + // later send time; deleting by id alone removed that newer payload, + // and its ZSET entry was then dropped as canceled — the send was lost. + const sendAt = drizzleSql`${scheduledPayloads.payload}->>'scheduledAt'`; + await db.delete(scheduledPayloads).where( + or( + ...dbCleanupIds.map((taskId) => { + const released = (payloadMap.get(taskId) as any)?.scheduledAt; + return and( + eq(scheduledPayloads.taskId, taskId), + typeof released === "string" + ? drizzleSql`${sendAt} = ${released}` + : drizzleSql`${sendAt} IS NULL`, + ); + }), + ), + ); } catch (err) { logger.error( { err }, @@ -284,7 +325,7 @@ export async function startSchedulerWorker() { { stream: STREAMS.SCHEDULED, group: CONSUMER_GROUPS.SCHEDULER, - consumer: `scheduler-${process.pid}`, + consumer: uniqueConsumerId("scheduler"), batchSize: config.WORKER_CONCURRENCY, }, )); diff --git a/src/services/scheduler/send-gate.ts b/src/services/scheduler/send-gate.ts new file mode 100644 index 0000000..738079f --- /dev/null +++ b/src/services/scheduler/send-gate.ts @@ -0,0 +1,117 @@ +import { and, eq, inArray } from "drizzle-orm"; +import { suppressions } from "@/db/schema.js"; +import { ContactRepository, TemplateRepository, UserRepository } from "@/repositories/index.js"; +import { normaliseTarget } from "@/shared/index.js"; + +export type BlockReason = + "suppressed" | "recipient_not_found" | "user_opted_out" | "channel_disabled" | "contact_inactive"; + +/** The fields of a parked `notification.dispatched` payload the gate reads. */ +interface ParkedTask { + taskId: string; + projectId: string; + recipientId: string; + channel: string; + templateId?: string; + destination?: string; +} + +/** + * Re-checks parked sends at the moment they are released. + * + * The engine clears a message against suppressions, opt-outs and contacts when + * it first sees it. A send parked for later — a `sendAt`, a quiet-hours + * deferral, a retry — can sit for hours or days, and whatever the person did in + * between (unsubscribed, complained, hard-bounced, removed the address) must + * still stop it. Without this the scheduler sent to them anyway. + * + * Reads Postgres directly rather than through the repositories' caches: this + * runs once per poll, not once per message, and a stale "still subscribed" is + * the very thing it exists to prevent. + * + * Returns the blocked tasks with the reason. Throws if a lookup fails, so the + * caller holds the batch rather than sending it unchecked. + */ +export async function findBlockedAtRelease( + db: any, + tasks: ParkedTask[], +): Promise> { + const blocked = new Map(); + + const byProject = new Map(); + for (const t of tasks) { + if (!t?.projectId || !t.recipientId) continue; + if (!byProject.has(t.projectId)) byProject.set(t.projectId, []); + byProject.get(t.projectId)!.push(t); + } + + for (const [projectId, group] of byProject) { + const recipientIds = [...new Set(group.map((t) => t.recipientId))]; + const templateIds = [...new Set(group.map((t) => t.templateId).filter(Boolean))] as string[]; + const targets = [ + ...new Set(group.map((t) => t.destination && normaliseTarget(t.destination)).filter(Boolean)), + ] as string[]; + + const templateRepo = new TemplateRepository(db); + const [suppressedRows, users, contactsByUser, templates] = await Promise.all([ + targets.length > 0 + ? db + .select({ channel: suppressions.channel, target: suppressions.target }) + .from(suppressions) + .where( + and(eq(suppressions.projectId, projectId), inArray(suppressions.target, targets)), + ) + : [], + new UserRepository(db).findRecordsByIds(projectId, recipientIds), + new ContactRepository(db).findActiveByUserIds(projectId, recipientIds), + Promise.all(templateIds.map((id) => templateRepo.findById(projectId, id))), + ]); + + const suppressed = new Set( + (suppressedRows as { channel: string; target: string }[]).map( + (r) => `${r.channel}:${normaliseTarget(r.target)}`, + ), + ); + const userById = new Map(users.map((u) => [u.userId, u])); + const topicsByTemplate = new Map(); + templateIds.forEach((id, i) => topicsByTemplate.set(id, templates[i]?.topics ?? [])); + + for (const t of group) { + const topics = t.templateId ? (topicsByTemplate.get(t.templateId) ?? []) : []; + const reason = decide(t, topics, suppressed, userById.get(t.recipientId), contactsByUser); + if (reason) blocked.set(t.taskId, reason); + } + } + + return blocked; +} + +function decide( + t: ParkedTask, + topics: string[], + suppressed: Set, + user: + | { preferences: { topics?: Record; channels?: Record } } + | undefined, + contactsByUser: Map, +): BlockReason | undefined { + // Checked first and for every priority, as the engine does: a suppression + // records that the address asked us to stop or is dead. + if (t.destination && suppressed.has(`${t.channel}:${normaliseTarget(t.destination)}`)) { + return "suppressed"; + } + if (!user) return "recipient_not_found"; + if (topics.some((topic) => user.preferences.topics?.[topic] === false)) return "user_opted_out"; + if (user.preferences.channels?.[t.channel] === false) return "channel_disabled"; + + if (t.destination) { + const contact = (contactsByUser.get(t.recipientId) ?? []).find( + (c) => c.channel === t.channel && c.target === t.destination, + ); + if (!contact) return "contact_inactive"; + if (topics.some((topic) => contact.preferences?.topics?.[topic] === false)) { + return "user_opted_out"; + } + } + return undefined; +} diff --git a/src/services/workflow/main.ts b/src/services/workflow/main.ts index c8d075d..d7dd90d 100644 --- a/src/services/workflow/main.ts +++ b/src/services/workflow/main.ts @@ -26,7 +26,12 @@ import { } from "@/workflows/index.js"; import { type WorkerOptions } from "@/shared/index.js"; import { startHealthReporter } from "@/workers/index.js"; -import { createStreamConsumer, createWorkerRuntime, shutdownWorker } from "@/workers/bootstrap.js"; +import { + createStreamConsumer, + createWorkerRuntime, + shutdownWorker, + uniqueConsumerId, +} from "@/workers/bootstrap.js"; /** How long a workflow instance lock is held before it self-expires. */ const WORKFLOW_LOCK_TTL_SECONDS = 60; @@ -34,6 +39,11 @@ const WORKFLOW_LOCK_TTL_SECONDS = 60; const WORKFLOW_LOCK_RENEW_MS = (WORKFLOW_LOCK_TTL_SECONDS / 3) * 1000; /** How long a claimed workflow timer stays invisible to other pollers. */ const WORKFLOW_TIMER_VISIBILITY_MS = 60_000; +/** Wake-ups for sleeping instances: waits, event timeouts, deferred resumes. */ +export const WORKFLOW_TIMERS_KEY = "notif:workflow:timers"; +/** How soon a resume that found its instance busy is tried again. */ +const WORKFLOW_RESUME_RETRY_MS = 2_000; +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; loadEnv(); const config = readBaseConfig(); @@ -119,6 +129,25 @@ export class WorkflowWorker extends BaseWorker { } } + /** + * Re-delivers a resume through the timer ZSET a few seconds from now. The + * poll loop publishes it once the instance is pending again, and drops it if + * the instance has finished. The member carries no timestamp, so repeated + * deferrals of one instance collapse into a single entry. + */ + private async deferResume(payload: any): Promise { + await this.redisCli.zadd( + WORKFLOW_TIMERS_KEY, + Date.now() + WORKFLOW_RESUME_RETRY_MS, + JSON.stringify({ + instanceId: payload.instanceId, + name: payload.name, + projectId: payload.projectId, + input: payload.input ?? {}, + }), + ); + } + async process(message: StreamMessage): Promise { const { event } = message; const publishPromises: Promise[] = []; @@ -189,7 +218,12 @@ export class WorkflowWorker extends BaseWorker { "NX", ); if (!acquired) { - this.logger.info({ instanceId }, "Workflow is locked by another process, skipping"); + // Another process is mid-run on this instance — typically still + // persisting the very suspension this resume answers, when the awaited + // event arrived quickly. Acking here dropped the resume, and with the + // waiter already consumed nothing would ever wake the instance again. + this.logger.info({ instanceId }, "Workflow is locked by another process, deferring resume"); + await this.deferResume(payload); return; } @@ -225,6 +259,13 @@ export class WorkflowWorker extends BaseWorker { instance = rows[0]!; } + if (instance.status === "running") { + // Its last holder died mid-run (the reaper will set it back to + // pending) or has not finished suspending. Either way this resume + // is still owed. + await this.deferResume(payload); + return; + } if (instance.status !== "pending") { this.logger.info({ instanceId }, "Workflow is not pending, skipping"); return; @@ -259,6 +300,9 @@ export class WorkflowWorker extends BaseWorker { instance!.input, payload.projectId, `wf-${instanceId}-${stepId}`, + // The contract wants a uuid, which the API always issues; anything + // else would fail validation and drop the notification. + UUID_RE.test(instanceId) ? instanceId : undefined, ); const result = await new Promise<{ messageId: string; notificationId: string }>( @@ -295,7 +339,15 @@ export class WorkflowWorker extends BaseWorker { }, wait: async (duration) => { const stepId = String(currentStepIndex++); - if (stepOutputMap.has(stepId)) return; + if (stepOutputMap.has(stepId)) { + // A resume that arrives early — a duplicate trigger, a replayed + // message — must not cut the sleep short. The timer is still set. + const until = Number(stepOutputMap.get(stepId)?.scheduledAt); + if (Number.isFinite(until) && until > Date.now()) { + throw new SuspendExecutionError("wait", { duration }); + } + return; + } // Simple duration parse (e.g. '2h' -> ms) let ms = 0; @@ -311,7 +363,7 @@ export class WorkflowWorker extends BaseWorker { // If the process dies after this point the source event is retried; // if it dies after the database write, the timer is already durable. await this.redisCli.zadd( - "notif:workflow:timers", + WORKFLOW_TIMERS_KEY, resumeAt, JSON.stringify({ instanceId: instance!.id, @@ -335,7 +387,12 @@ export class WorkflowWorker extends BaseWorker { const stepId = String(currentStepIndex++); if (stepOutputMap.has(stepId)) { const out = stepOutputMap.get(stepId); - if (out && typeof out === "object" && (out as any).timedOut === true) { + // Still waiting: neither the event nor the timeout has filled the + // step in, so this resume is spurious. + if (out === null || out === undefined) { + throw new SuspendExecutionError("waitForEvent", { eventName }); + } + if (typeof out === "object" && (out as any).timedOut === true) { return null; } return out; @@ -355,7 +412,7 @@ export class WorkflowWorker extends BaseWorker { const resumeAt = Date.now() + ms; await this.redisCli.zadd( - "notif:workflow:timers", + WORKFLOW_TIMERS_KEY, resumeAt, JSON.stringify({ instanceId: instance!.id, @@ -368,16 +425,8 @@ export class WorkflowWorker extends BaseWorker { }), ); - // Register waiter - await this.dbConn.insert(workflowWaiters).values({ - instanceId: instance!.id, - projectId: payload.projectId, - eventName: eventName, - matchCriteria: options.match, - expiresAt: new Date(resumeAt), - }); - - // Register step as pending event + // The pending step before the waiter: an event that matches the + // waiter the moment it exists must find a step to record itself in. await this.dbConn.insert(workflowSteps).values({ instanceId: instance!.id, projectId: payload.projectId, @@ -386,6 +435,14 @@ export class WorkflowWorker extends BaseWorker { output: null, // this will be updated by event worker or timeout }); + await this.dbConn.insert(workflowWaiters).values({ + instanceId: instance!.id, + projectId: payload.projectId, + eventName: eventName, + matchCriteria: options.match, + expiresAt: new Date(resumeAt), + }); + throw new SuspendExecutionError("waitForEvent", { eventName }); }, run: async (stepName, fn) => { @@ -466,7 +523,7 @@ export async function startWorkflowWorker() { { stream: STREAMS.WORKFLOW_INBOUND, group: CONSUMER_GROUPS.WORKFLOW, - consumer: `workflow-${process.pid}`, + consumer: uniqueConsumerId("workflow"), batchSize: config.WORKER_CONCURRENCY, }, )); @@ -488,7 +545,7 @@ export async function startWorkflowWorker() { try { const now = Date.now(); const tasks = await redis.native.schedulerPoll( - "notif:workflow:timers", + WORKFLOW_TIMERS_KEY, now, 100, WORKFLOW_TIMER_VISIBILITY_MS, @@ -505,8 +562,13 @@ export async function startWorkflowWorker() { .where(eq(workflowInstances.id, task.instanceId)) .limit(1) )[0]; - if (!inst || inst.status === "completed" || inst.status === "failed") { - await redis.native.zrem("notif:workflow:timers", taskStr); + if ( + !inst || + inst.status === "completed" || + inst.status === "failed" || + inst.status === "canceled" + ) { + await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr); continue; } // A timer can be observed while the worker is still persisting the @@ -530,7 +592,7 @@ export async function startWorkflowWorker() { // If the waiter was already deleted (by EventWorker when event arrived before timeout), // this timer is stale. Clean up from Redis and skip re-resuming the workflow. if (deleted.length === 0) { - await redis.native.zrem("notif:workflow:timers", taskStr); + await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr); continue; } @@ -556,7 +618,7 @@ export async function startWorkflowWorker() { await workflowProducer.publish( buildStreamEvent("workflow.resumed", task, "scheduler", undefined), ); - await redis.native.zrem("notif:workflow:timers", taskStr); + await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr); logger.info({ instanceId: task.instanceId }, "Workflow resumed from timer"); } } catch (err) { diff --git a/src/transport/index.ts b/src/transport/index.ts index 42c0dd5..8192062 100644 --- a/src/transport/index.ts +++ b/src/transport/index.ts @@ -10,6 +10,12 @@ export interface DeliveryResult { providerMessageId?: string; invalidToken?: boolean; error?: string; + /** + * `false` marks a failure another attempt cannot fix — a malformed address, + * a rejected sender. Anything else (an outage, a timeout, a 5xx) is retried + * up to the task's `maxAttempts`. + */ + retryable?: boolean; } export interface WebhookEvent { diff --git a/src/workers/index.ts b/src/workers/index.ts index b91b196..b1232bc 100644 --- a/src/workers/index.ts +++ b/src/workers/index.ts @@ -220,7 +220,7 @@ export abstract class BaseWorker { { err, eventId: event.id }, "non-retryable error on inline message — moving to dead-letter queue", ); - await this.consumer.deadLetter(event); + await this.consumer.deadLetter(event, (err as Error).message); globalEmitter.emit("notification:failed", event.id, (err as Error).message, event.type); return; } @@ -239,7 +239,12 @@ export abstract class BaseWorker { { messageId: message.id, retryCount }, "max retries exceeded, moving to dead-letter queue", ); - await this.consumer.nack(message.id, message.event, stream); + await this.consumer.nack( + message.id, + message.event, + stream, + `max retries exceeded (${retryCount - 1} attempts)`, + ); globalEmitter.emit( "notification:failed", message.id, @@ -285,7 +290,7 @@ export abstract class BaseWorker { { err, messageId: message.id }, "non-retryable error encountered, immediately moving to dead-letter queue without retry loop", ); - await this.consumer.nack(message.id, message.event, stream); + await this.consumer.nack(message.id, message.event, stream, (err as Error).message); globalEmitter.emit( "notification:failed", message.id, diff --git a/src/workflows/sdk.ts b/src/workflows/sdk.ts index 4ef5d36..367653c 100644 --- a/src/workflows/sdk.ts +++ b/src/workflows/sdk.ts @@ -90,6 +90,7 @@ export function buildStepNotifyPayload( instanceInput: unknown, projectId: string, idempotencyKey?: string, + workflowInstanceId?: string, ): NotificationRequestedPayload { return { projectId, @@ -102,5 +103,6 @@ export function buildStepNotifyPayload( fallback: args.fallback ?? false, scheduledAt: args.sendAt, idempotencyKey, + ...(workflowInstanceId ? { workflowInstanceId } : {}), }; } diff --git a/tests/api-admin.test.ts b/tests/api-admin.test.ts index 669652a..1547e55 100644 --- a/tests/api-admin.test.ts +++ b/tests/api-admin.test.ts @@ -427,7 +427,11 @@ describe("API operational handlers", () => { ); expect(res.statusCode).toBe(200); - expect(parse(res)).toEqual({ success: true, replayedId: "1-0" }); + expect(parse(res)).toEqual({ + success: true, + replayedId: "1-0", + stream: STREAMS.INBOUND_CRITICAL, + }); expect(deps.redis.native.xadd).toHaveBeenCalledWith( STREAMS.INBOUND_CRITICAL, "*", @@ -534,10 +538,11 @@ describe("API operational handlers", () => { it("cancelWorkflow returns 204 once the instance is canceled", async () => { const res = createMockRes(); - await handlers.cancelWorkflow(createMockReq(), res, ctx({ params: { id: "wf_1" } })); + const id = "33333333-3333-4333-8333-333333333333"; + await handlers.cancelWorkflow(createMockReq(), res, ctx({ params: { id } })); expect(res.statusCode).toBe(204); - expect(deps.workflowRepo.cancelInstance).toHaveBeenCalledWith("proj_1", "wf_1"); + expect(deps.workflowRepo.cancelInstance).toHaveBeenCalledWith("proj_1", id); }); it("cancelWorkflow 400s when the instance cannot be canceled", async () => { diff --git a/tests/api-suppressions.test.ts b/tests/api-suppressions.test.ts index c08a429..fbb13c1 100644 --- a/tests/api-suppressions.test.ts +++ b/tests/api-suppressions.test.ts @@ -380,6 +380,19 @@ describe("API compliance handlers", () => { describe("getCampaignStats", () => { const statsCtx = (campaign = "spring-sale") => ctx({ params: { campaign } }); + /** + * The grouped rows the stats query returns for `sent` tasks on a channel, + * shaped like the real delivery log: every attempt writes a `dispatched` + * row and then an `attempt` row carrying its outcome. + */ + const sends = (channel: string, delivered: number, failed = 0) => [ + { channel, kind: "dispatched", status: "dispatched", tasks: String(delivered + failed) }, + ...(delivered + ? [{ channel, kind: "attempt", status: "delivered", tasks: String(delivered) }] + : []), + ...(failed ? [{ channel, kind: "attempt", status: "failed", tasks: String(failed) }] : []), + ]; + it("400s when no campaign is named", async () => { const res = createMockRes(); await handlers.getCampaignStats(createMockReq(), res, ctx({ params: {} })); @@ -399,10 +412,7 @@ describe("API compliance handlers", () => { }); it("splits delivery rows into sent, delivered and failed", async () => { - mockDb.queueSelect([ - { channel: "email", kind: "delivery", status: "delivered", tasks: "90" }, - { channel: "email", kind: "delivery", status: "failed", tasks: "10" }, - ]); + mockDb.queueSelect(sends("email", 90, 10)); const res = createMockRes(); await handlers.getCampaignStats(createMockReq(), res, statsCtx()); @@ -416,7 +426,7 @@ describe("API compliance handlers", () => { it("counts engagement kinds separately from the send total", async () => { mockDb.queueSelect([ - { channel: "email", kind: "delivery", status: "delivered", tasks: "100" }, + ...sends("email", 100), { channel: "email", kind: "opened", status: "opened", tasks: "40" }, { channel: "email", kind: "clicked", status: "clicked", tasks: "10" }, ]); @@ -434,7 +444,7 @@ describe("API compliance handlers", () => { it("rounds a rate to two decimals", async () => { mockDb.queueSelect([ - { channel: "email", kind: "delivery", status: "delivered", tasks: "3" }, + ...sends("email", 3), { channel: "email", kind: "opened", status: "opened", tasks: "1" }, ]); const res = createMockRes(); @@ -445,7 +455,7 @@ describe("API compliance handlers", () => { }); it("reports a rate as null, not zero, when its denominator is zero", async () => { - mockDb.queueSelect([{ channel: "email", kind: "delivery", status: "failed", tasks: "5" }]); + mockDb.queueSelect(sends("email", 0, 5)); const res = createMockRes(); await handlers.getCampaignStats(createMockReq(), res, statsCtx()); @@ -458,11 +468,7 @@ describe("API compliance handlers", () => { }); it("breaks the funnel down per channel and totals across them", async () => { - mockDb.queueSelect([ - { channel: "email", kind: "delivery", status: "delivered", tasks: "50" }, - { channel: "sms", kind: "delivery", status: "delivered", tasks: "30" }, - { channel: "sms", kind: "delivery", status: "failed", tasks: "20" }, - ]); + mockDb.queueSelect([...sends("email", 50), ...sends("sms", 30, 20)]); const res = createMockRes(); await handlers.getCampaignStats(createMockReq(), res, statsCtx()); @@ -476,7 +482,7 @@ describe("API compliance handlers", () => { }); it("warns that opens are untracked rather than zero on a channel that cannot report them", async () => { - mockDb.queueSelect([{ channel: "sms", kind: "delivery", status: "delivered", tasks: "10" }]); + mockDb.queueSelect(sends("sms", 10)); const res = createMockRes(); await handlers.getCampaignStats(createMockReq(), res, statsCtx()); @@ -487,9 +493,7 @@ describe("API compliance handlers", () => { }); it("warns when email was delivered but no engagement ever arrived", async () => { - mockDb.queueSelect([ - { channel: "email", kind: "delivery", status: "delivered", tasks: "10" }, - ]); + mockDb.queueSelect(sends("email", 10)); const res = createMockRes(); await handlers.getCampaignStats(createMockReq(), res, statsCtx()); @@ -501,7 +505,7 @@ describe("API compliance handlers", () => { it("stays quiet when engagement is both possible and present", async () => { mockDb.queueSelect([ - { channel: "email", kind: "delivery", status: "delivered", tasks: "10" }, + ...sends("email", 10), { channel: "email", kind: "opened", status: "opened", tasks: "4" }, ]); const res = createMockRes(); diff --git a/tests/api.test.ts b/tests/api.test.ts index 45dba5e..43cfb8f 100644 --- a/tests/api.test.ts +++ b/tests/api.test.ts @@ -173,6 +173,9 @@ describe("API Handlers", () => { listApiKeys: vi.fn().mockResolvedValue([]), deleteApiKey: vi.fn().mockResolvedValue(true), updateApiKeyHash: vi.fn().mockResolvedValue(true), + findThrottleSettings: vi + .fn() + .mockResolvedValue({ throttleLimit: null, throttleWindowHours: null }), } as any, workflowRepo: { listDefinitions: vi.fn().mockResolvedValue([]), @@ -509,6 +512,10 @@ describe("API Handlers", () => { }); describe("New CRUD Endpoints", () => { + const PROJECT_UUID = "11111111-1111-4111-8111-111111111111"; + const KEY_UUID = "22222222-2222-4222-8222-222222222222"; + const WF_ID = "33333333-3333-4333-8333-333333333333"; + it("listWorkflows", async () => { const res = createMockRes(); await handlers.listWorkflows({} as any, res, { @@ -525,11 +532,11 @@ describe("API Handlers", () => { const res = createMockRes(); await handlers.getWorkflow({} as any, res, { projectId: "proj_1", - params: { id: "wf_1" }, + params: { id: WF_ID }, query: new URLSearchParams(), } as any); expect(res.statusCode).toBe(200); - expect(deps.workflowRepo.getInstance).toHaveBeenCalledWith("proj_1", "wf_1"); + expect(deps.workflowRepo.getInstance).toHaveBeenCalledWith("proj_1", WF_ID); }); it("listUsers", async () => { @@ -590,11 +597,11 @@ describe("API Handlers", () => { const res = createMockRes(); await handlers.updateProject(req, res, { projectId: "proj_1", - params: { id: "proj_1" }, + params: { id: PROJECT_UUID }, query: new URLSearchParams(), } as any); expect(res.statusCode).toBe(200); - expect(deps.projectRepo.updateSettings).toHaveBeenCalledWith("proj_1", { + expect(deps.projectRepo.updateSettings).toHaveBeenCalledWith(PROJECT_UUID, { rateLimitRpm: 1000, }); }); @@ -603,11 +610,11 @@ describe("API Handlers", () => { const res = createMockRes(); await handlers.deleteProject({} as any, res, { projectId: "proj_1", - params: { id: "proj_1" }, + params: { id: PROJECT_UUID }, query: new URLSearchParams(), } as any); expect(res.statusCode).toBe(204); - expect(deps.projectRepo.delete).toHaveBeenCalledWith("proj_1"); + expect(deps.projectRepo.delete).toHaveBeenCalledWith(PROJECT_UUID); }); it("createProjectKey", async () => { @@ -616,7 +623,7 @@ describe("API Handlers", () => { (deps.projectRepo.createApiKey as any).mockResolvedValue({ id: "key_1" }); await handlers.createProjectKey(req, res, { projectId: "proj_1", - params: { id: "proj_1" }, + params: { id: PROJECT_UUID }, query: new URLSearchParams(), } as any); expect(res.statusCode).toBe(201); @@ -630,11 +637,11 @@ describe("API Handlers", () => { (deps.projectRepo.listApiKeys as any).mockResolvedValue([{ id: "key_1", role: "admin" }]); await handlers.listProjectKeys({ method: "GET" } as any, res, { projectId: "proj_1", - params: { id: "proj_1" }, + params: { id: PROJECT_UUID }, query: new URLSearchParams(), } as any); expect(res.statusCode).toBe(200); - expect(deps.projectRepo.listApiKeys).toHaveBeenCalledWith("proj_1"); + expect(deps.projectRepo.listApiKeys).toHaveBeenCalledWith(PROJECT_UUID); }); it("deleteProjectKey", async () => { @@ -642,11 +649,11 @@ describe("API Handlers", () => { (deps.projectRepo.deleteApiKey as any).mockResolvedValue(true); await handlers.deleteProjectKey({ method: "DELETE" } as any, res, { projectId: "proj_1", - params: { id: "proj_1", keyId: "key_1" }, + params: { id: PROJECT_UUID, keyId: KEY_UUID }, query: new URLSearchParams(), } as any); expect(res.statusCode).toBe(204); - expect(deps.projectRepo.deleteApiKey).toHaveBeenCalledWith("proj_1", "key_1"); + expect(deps.projectRepo.deleteApiKey).toHaveBeenCalledWith(PROJECT_UUID, KEY_UUID); }); it("deleteProjectKey returns 404 if key does not exist", async () => { diff --git a/tests/chaos/crash.test.ts b/tests/chaos/crash.test.ts index 313166d..b1434fd 100644 --- a/tests/chaos/crash.test.ts +++ b/tests/chaos/crash.test.ts @@ -26,23 +26,21 @@ describe("Crash Testing (Chaos Monkey)", () => { const spawnWorker = (type: string, id: string) => { logger.info(`Spawning worker: ${type} (${id})`); - const isWin = process.platform === "win32"; - const cp = spawn( - isWin ? "cmd.exe" : "npx", - isWin - ? ["/c", "npx", "--no-install", "tsx", "tests/chaos/worker-runner.ts"] - : ["--no-install", "tsx", "tests/chaos/worker-runner.ts"], - { - env: { - ...process.env, - WORKER_TYPE: type, - DATABASE_URL: dbUrl, - REDIS_URL: redisUrl, - LOG_LEVEL: "silent", - }, - stdio: "ignore", // ignore output to keep logs clean + // The worker is this node binary's own child, with tsx loaded as an import + // hook. Launched through `cmd /c npx tsx` (or `npx tsx`) instead, a kill + // reached only the wrapper: on Windows the worker itself survived every + // "crash", so no crash was ever tested, and each respawn leaked another + // worker until the machine ran out of memory. + const cp = spawn(process.execPath, ["--import", "tsx", "tests/chaos/worker-runner.ts"], { + env: { + ...process.env, + WORKER_TYPE: type, + DATABASE_URL: dbUrl, + REDIS_URL: redisUrl, + LOG_LEVEL: "silent", }, - ); + stdio: "ignore", // ignore output to keep logs clean + }); cp.on("exit", (code, signal) => { logger.info(`Worker ${type} (${id}) exited with signal ${signal} (code ${code})`); diff --git a/tests/client.test.ts b/tests/client.test.ts index 3df9375..01e5e76 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -2,16 +2,13 @@ import { describe, it, expect, vi, afterEach } from "vitest"; import { NotifkitClient } from "@/client.js"; /** - * Stubs `fetch` with a minimal Response double. `status` drives `ok` unless it - * is given explicitly, which lets the error-path tests cover a 4xx that still - * carries a JSON body. + * Stubs `fetch` with a real `Response` carrying `body` as JSON, so the client + * reads it the way it reads a server's. The request/response round trip itself + * is covered against a running server in tests/integration/api.test.ts; these + * pin down what the client puts on the wire. */ function stubFetch(body: unknown, status = 200) { - const fn = vi.fn().mockResolvedValue({ - status, - ok: status >= 200 && status < 300, - json: async () => body, - }); + const fn = vi.fn().mockImplementation(async () => new Response(JSON.stringify(body), { status })); vi.stubGlobal("fetch", fn); return fn; } @@ -76,18 +73,20 @@ describe("NotifkitClient", () => { }); it("returns undefined on 204 without parsing a body", async () => { - const fetchMock = vi.fn().mockResolvedValue({ - status: 204, - ok: true, - json: async () => { - throw new Error("204 responses have no body to parse"); - }, - }); - vi.stubGlobal("fetch", fetchMock); - + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(null, { status: 204 }))); await expect(client().deleteUser("usr_1")).resolves.toBeUndefined(); }); + it("reports the status when an error response is not JSON", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(new Response("Bad Gateway", { status: 502 })), + ); + await expect(client().addUser({ id: "usr_1" })).rejects.toThrow( + "Request failed with status 502", + ); + }); + it("sends no body when none is supplied", async () => { const fetchMock = stubFetch({ workflows: [] }); await client().listWorkflows(); diff --git a/tests/integration/ai.test.ts b/tests/integration/ai.test.ts new file mode 100644 index 0000000..4524f84 --- /dev/null +++ b/tests/integration/ai.test.ts @@ -0,0 +1,209 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { randomUUID } from "node:crypto"; +import { MockLanguageModelV4 } from "ai/test"; +import { APICallError } from "ai"; +import { STREAMS } from "@/contracts/index.js"; +import { useInfra, waitFor, settle } from "./support/infra.js"; +import { RecordingTransport, startNotifkit, type Running, type Api } from "./support/pipeline.js"; + +/** + * AI-generated content, end to end. The language model is the one stand-in: + * it is an external paid API, so a local model records every prompt it is + * sent and answers deterministically. Everything else — engine, AI worker, + * delivery, Redis, Postgres — is real. + */ + +const infra = useInfra(); +let app: Running; +const email = new RecordingTransport("email"); + +const prompts: string[] = []; +let failNext: Error | null = null; + +const model = new MockLanguageModelV4({ + doGenerate: async (opts: any) => { + const prompt = opts.prompt.flatMap((m: any) => m.content.map((c: any) => c.text)).join(""); + prompts.push(prompt); + if (failNext) { + const err = failNext; + failNext = null; + throw err; + } + return { + content: [{ type: "text", text: `AI(${prompt})` }], + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 1, text: 1, reasoning: 0 }, + }, + warnings: [], + } as any; + }, +}); + +beforeAll(async () => { + app = await startNotifkit(infra, { + services: ["api", "enricher", "engine", "ai", "scheduler", "delivery", "events"], + providers: [email], + aiModel: model as any, + env: { RATE_LIMIT_PER_HOUR: "100000" }, + }); +}, 120_000); + +afterAll(async () => { + await app?.stop(); +}, 60_000); + +async function project() { + const p = await app.project(); + await p.api("PUT", "/v1/templates", { + templates: [ + { id: "digest", channel: "email", content: { subject: "Digest", text: "{{summary}}" } }, + { + id: "smart", + channel: "email", + content: { subject: "{{headline}}" }, + aiPrompts: { headline: "Write a headline about {{topic}}" }, + }, + ], + }); + return p; +} + +async function notify(api: Api, body: Record) { + const res = await api("POST", "/v1/notify", body); + expect(res.status).toBe(202); +} + +describe("AI-generated notifications", () => { + it("generates each prompt with the request's data and sends the result", async () => { + const p = await project(); + await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" }); + const marker = randomUUID().slice(0, 8); + + await notify(p.api, { + user: "u", + template: "digest", + data: { name: marker }, + aiPrompts: { summary: "Summarise the week for {{name}}" }, + }); + + await waitFor("delivered", () => email.for(p.id).length === 1); + expect(prompts).toContain(`Summarise the week for ${marker}`); + expect(email.for(p.id)[0]!.renderedContent.content).toEqual({ + subject: "Digest", + text: `AI(Summarise the week for ${marker})`, + }); + expect(email.for(p.id)[0]!.destination).toBe("u@x.com"); + }); + + it("uses the template's own prompts", async () => { + const p = await project(); + await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" }); + const topic = randomUUID().slice(0, 8); + await notify(p.api, { user: "u", template: "smart", data: { topic } }); + await waitFor("delivered", () => email.for(p.id).length === 1); + expect(email.for(p.id)[0]!.renderedContent.content).toEqual({ + subject: `AI(Write a headline about ${topic})`, + }); + }); + + it("generates once per address, sending each address its own copy", async () => { + const p = await project(); + await p.api("POST", "/v1/users", { id: "u", email: ["a@x.com", "b@x.com"] }); + await notify(p.api, { user: "u", template: "digest", aiPrompts: { summary: "hi" } }); + await waitFor("both delivered", () => email.for(p.id).length === 2); + expect( + email + .for(p.id) + .map((t) => t.destination) + .sort(), + ).toEqual(["a@x.com", "b@x.com"]); + }); + + it("never sends AI mail to a suppressed address", async () => { + const p = await project(); + await p.api("POST", "/v1/suppressions", { channel: "email", target: "gone@x.com" }); + await p.api("POST", "/v1/users", { id: "u", email: "gone@x.com" }); + await p.api("POST", "/v1/users", { id: "control", email: "control@x.com" }); + + await notify(p.api, { user: "u", template: "digest", aiPrompts: { summary: "hi" } }); + await notify(p.api, { user: "control", template: "digest", aiPrompts: { summary: "hi" } }); + + await waitFor("control delivered", () => email.for(p.id).length === 1); + await settle(1_000); + expect(email.for(p.id).map((t) => t.destination)).toEqual(["control@x.com"]); + }); + + it("sends once when an AI task is replayed after a crash lost its completion marker", async () => { + const p = await project(); + await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" }); + await notify(p.api, { user: "u", template: "digest", aiPrompts: { summary: "once" } }); + await waitFor("delivered", () => email.for(p.id).length === 1); + + // A crash between dispatching and recording completion: the marker never + // landed, and the pending entry is replayed. + const [entry] = (await infra.redis.xrange(STREAMS.AI_PENDING, "-", "+")).filter(([, f]) => + f[1]!.includes(p.id), + ); + for (const key of await infra.redis.keys("notif:processed:ai:*")) await infra.redis.del(key); + await infra.redis.xadd(STREAMS.AI_PENDING, "*", "data", entry![1][1]!); + + await settle(3_000); + expect(email.for(p.id)).toHaveLength(1); + }); + + it("fails a notification once, without retrying, when the model rejects the request", async () => { + const p = await project(); + await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" }); + const failures: unknown[][] = []; + const on = (...a: unknown[]) => failures.push(a); + app.server.on("notification:failed", on); + try { + const marker = `reject-${randomUUID().slice(0, 8)}`; + failNext = new APICallError({ + message: "invalid prompt", + url: "https://model", + requestBodyValues: {}, + statusCode: 400, + }); + await notify(p.api, { user: "u", template: "digest", aiPrompts: { summary: marker } }); + + await waitFor("failure reported", () => failures.length > 0); + await settle(1_000); + expect(prompts.filter((pr) => pr === marker)).toHaveLength(1); + expect(email.for(p.id)).toEqual([]); + } finally { + app.server.off("notification:failed", on); + } + }); + + it("caps the number of prompts run for one notification", async () => { + const p = await project(); + await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" }); + const tag = randomUUID().slice(0, 8); + const aiPrompts = Object.fromEntries( + Array.from({ length: 8 }, (_, i) => [`k${i}`, `${tag}-${i}`]), + ); + await notify(p.api, { user: "u", template: "digest", aiPrompts }); + await waitFor("delivered", () => email.for(p.id).length === 1); + expect(prompts.filter((pr) => pr.startsWith(tag))).toHaveLength(5); + }); + + it("holds a scheduled AI notification until its time", async () => { + const p = await project(); + await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" }); + await notify(p.api, { + user: "u", + template: "digest", + aiPrompts: { summary: "later" }, + sendAt: new Date(Date.now() + 3_000).toISOString(), + }); + await waitFor( + "parked", + async () => (await p.api("GET", "/v1/notifications/scheduled")).body.scheduled?.length === 1, + ); + expect(email.for(p.id)).toEqual([]); + await waitFor("sent", () => email.for(p.id).length === 1, 20_000); + }, 30_000); +}); diff --git a/tests/integration/api.test.ts b/tests/integration/api.test.ts new file mode 100644 index 0000000..0d6dda2 --- /dev/null +++ b/tests/integration/api.test.ts @@ -0,0 +1,1066 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; +import { createHash, randomUUID } from "node:crypto"; +import { NotifkitClient } from "@/client.js"; +import { signUnsubscribeToken } from "@/unsubscribe/index.js"; +import { hashPassword } from "@/services/auth/index.js"; +import { AdminUserRepository } from "@/repositories/index.js"; +import { STREAMS } from "@/contracts/index.js"; +import type { Transport, WebhookEvent } from "@/transport/index.js"; +import { useInfra, waitFor, settle } from "./support/infra.js"; +import { + ADMIN_KEY, + RecordingTransport, + apiClient, + startNotifkit, + type Running, +} from "./support/pipeline.js"; + +/** + * The HTTP API as a caller sees it: status codes, bodies, auth, tenancy, and + * the side effects a request is supposed to have. Driven through the published + * SDK client where it has a method, and raw HTTP where the point is the wire. + */ + +const infra = useInfra(); +let app: Running; +const SECRET = "api-test-unsubscribe-secret"; + +// A provider with a webhook, as Resend or Twilio would mount one. +class WebhookTransport extends RecordingTransport implements Transport { + readonly webhookPath = "/webhooks/test"; + constructor() { + super("email"); + } + verifyWebhookChallenge(query: URLSearchParams) { + return query.get("hub.verify_token") === "ok" ? (query.get("hub.challenge") ?? "") : undefined; + } + verifyWebhook(_raw: string, headers: Record) { + return headers["x-signature"] === "valid"; + } + async parseWebhook(body: any): Promise { + return body.events; + } +} +const email = new WebhookTransport(); + +beforeAll(async () => { + app = await startNotifkit(infra, { + services: ["api", "enricher", "engine", "delivery", "scheduler", "events"], + providers: [email], + env: { + RATE_LIMIT_PER_HOUR: "100000", + UNSUBSCRIBE_SECRET: SECRET, + CORS_ORIGIN: "https://dash.example.com", + }, + }); +}, 120_000); + +afterAll(async () => { + await app?.stop(); +}, 60_000); + +/** Per-address counters for the public login and unsubscribe routes. */ +async function clearRateLimits() { + const keys = [ + ...(await infra.redis.keys("rate-limit:api:unsub:*")), + ...(await infra.redis.keys("rate-limit:api:auth:*")), + ]; + if (keys.length > 0) await infra.redis.del(...keys); +} + +function sdk(apiKey: string, headers: Record = {}) { + return new NotifkitClient({ baseUrl: app.baseUrl, apiKey, headers }); +} + +async function project() { + const p = await app.project(); + return { ...p, client: sdk(p.apiKey) }; +} + +// ─── Authentication & tenancy ─────────────────────────────────────────────── + +describe("authentication", () => { + it("rejects requests without a key, with an unknown key, or with a wrong admin key", async () => { + const anon = apiClient(app.baseUrl); + expect((await anon("GET", "/v1/users")).status).toBe(401); + expect((await apiClient(app.baseUrl, "nk_live_nope")("GET", "/v1/users")).status).toBe(401); + expect((await apiClient(app.baseUrl, "wrong-admin")("GET", "/v1/projects")).status).toBe(401); + expect((await anon("GET", "/v1/projects")).status).toBe(401); + }); + + it("accepts the key as a bearer token or an x-api-key header", async () => { + const p = await project(); + expect((await apiClient(app.baseUrl, p.apiKey)("GET", "/v1/segments")).status).toBe(200); + const viaHeader = apiClient(app.baseUrl, undefined, { "x-api-key": p.apiKey }); + expect((await viaHeader("GET", "/v1/segments")).status).toBe(200); + }); + + it("lets the admin key act on a project named in x-project-id, and requires one", async () => { + const p = await project(); + await p.client.addUser({ id: "u", email: "u@x.com" }); + expect((await app.admin("GET", "/v1/users")).status).toBe(400); + const asAdmin = sdk(ADMIN_KEY, { "x-project-id": p.id }); + expect((await asAdmin.getUser("u")).userId).toBe("u"); + }); + + it("keeps each project's data invisible to other projects", async () => { + const a = await project(); + const b = await project(); + await a.client.addUser({ id: "secret", email: "s@x.com" }); + await expect(b.client.getUser("secret")).rejects.toThrow(/user_not_found/); + expect((await b.client.listUsers()).users).toEqual([]); + }); + + it("forbids writes with a read-only key but allows reads", async () => { + const p = await project(); + const key = await app.admin("POST", `/v1/projects/${p.id}/keys`, { role: "read_only" }); + expect(key.status).toBe(201); + const ro = apiClient(app.baseUrl, key.body.apiKey); + expect((await ro("GET", "/v1/users")).status).toBe(200); + expect((await ro("POST", "/v1/users", { id: "x" })).status).toBe(403); + }); + + it("stops accepting a key as soon as it is deleted", async () => { + const p = await project(); + const key = await app.admin("POST", `/v1/projects/${p.id}/keys`, {}); + const client = apiClient(app.baseUrl, key.body.apiKey); + expect((await client("GET", "/v1/segments")).status).toBe(200); // now cached + + const del = await app.admin("DELETE", `/v1/projects/${p.id}/keys/${key.body.id}`); + expect(del.status).toBe(204); + await waitFor( + "revoked", + async () => (await client("GET", "/v1/segments")).status === 401, + 5_000, + ); + }); + + it("enforces a project's request rate limit", async () => { + const p = await project(); + await app.admin("PATCH", `/v1/projects/${p.id}`, { rateLimitRpm: 3 }); + await settle(200); // the auth cache drops the old limit over pub/sub + const statuses: number[] = []; + for (let i = 0; i < 5; i++) statuses.push((await p.api("GET", "/v1/segments")).status); + expect(statuses.filter((s) => s === 200)).toHaveLength(3); + const limited = await p.api("GET", "/v1/segments"); + expect(limited.status).toBe(429); + expect(limited.headers.get("retry-after")).toBe("60"); + }); + + describe("admin sessions", () => { + it("logs in, reads the session, reaches admin routes, and logs out", async () => { + await clearRateLimits(); + await new AdminUserRepository(infra.db).create({ + email: "ops@example.com", + username: "ops", + passwordHash: await hashPassword("correct horse"), + }); + const anon = apiClient(app.baseUrl, undefined, { "x-forwarded-for": randomUUID() }); + + expect( + (await anon("POST", "/v1/auth/login", { identifier: "ops", password: "wrong" })).status, + ).toBe(401); + expect( + (await anon("POST", "/v1/auth/login", { identifier: "nobody", password: "x" })).status, + ).toBe(401); + expect((await anon("POST", "/v1/auth/login", {})).status).toBe(400); + + const login = await anon("POST", "/v1/auth/login", { + identifier: "OPS@example.com", + password: "correct horse", + }); + expect(login.status).toBe(200); + expect(login.body.user).toMatchObject({ email: "ops@example.com", username: "ops" }); + const session = apiClient(app.baseUrl, login.body.token); + + expect((await session("GET", "/v1/auth/me")).body.user.email).toBe("ops@example.com"); + expect((await session("GET", "/v1/projects")).status).toBe(200); + expect((await session("GET", "/v1/system/health")).status).toBe(200); + + expect((await session("POST", "/v1/auth/logout")).status).toBe(200); + expect((await session("GET", "/v1/auth/me")).status).toBe(401); + expect((await session("GET", "/v1/projects")).status).toBe(401); + expect((await anon("GET", "/v1/auth/me")).status).toBe(401); + }); + + it("throttles repeated login attempts from one address", async () => { + await clearRateLimits(); + const statuses: number[] = []; + for (let i = 0; i < 12; i++) { + statuses.push( + ( + await apiClient(app.baseUrl)("POST", "/v1/auth/login", { + identifier: "x", + password: "y", + }) + ).status, + ); + } + expect(statuses.slice(0, 10).every((s) => s === 401)).toBe(true); + expect(statuses.slice(10)).toEqual([429, 429]); + }); + }); +}); + +// ─── Projects & keys ──────────────────────────────────────────────────────── + +describe("projects", () => { + it("creates, lists, updates and deletes projects", async () => { + const created = await app.admin("POST", "/v1/projects", { name: "Acme" }); + expect(created.status).toBe(201); + expect(created.body.apiKey).toMatch(/^nk_live_[0-9a-f]{64}$/); + const [row] = + await infra.sql`SELECT key_hash FROM project_api_keys WHERE project_id = ${created.body.id}`; + expect(row!.key_hash).toBe(createHash("sha256").update(created.body.apiKey).digest("hex")); + + expect((await app.admin("POST", "/v1/projects", {})).status).toBe(400); + const list = await sdk(ADMIN_KEY).listProjects(); + expect(list.projects.map((p) => p.id)).toContain(created.body.id); + + expect( + (await app.admin("PATCH", `/v1/projects/${created.body.id}`, { throttleLimit: "x" })).status, + ).toBe(400); + await sdk(ADMIN_KEY).updateProject(created.body.id, { throttleLimit: 7 }); + const [settings] = + await infra.sql`SELECT throttle_limit FROM projects WHERE id = ${created.body.id}`; + expect(settings!.throttle_limit).toBe(7); + + await sdk(ADMIN_KEY).deleteProject(created.body.id); + expect((await apiClient(app.baseUrl, created.body.apiKey)("GET", "/v1/users")).status).toBe( + 401, + ); + }); + + it("answers 404, not 500, for projects that do not exist or ids that are not ids", async () => { + const missing = randomUUID(); + expect((await app.admin("PATCH", `/v1/projects/${missing}`, { throttleLimit: 1 })).status).toBe( + 404, + ); + expect((await app.admin("DELETE", `/v1/projects/${missing}`)).status).toBe(404); + expect((await app.admin("POST", `/v1/projects/${missing}/keys`, {})).status).toBe(404); + expect((await app.admin("DELETE", `/v1/projects/${missing}/keys/${randomUUID()}`)).status).toBe( + 404, + ); + expect((await app.admin("DELETE", "/v1/projects/not-a-uuid")).status).toBe(404); + expect((await app.admin("GET", "/v1/projects/not-a-uuid/keys")).status).toBe(404); + }); + + it("lists a project's keys without revealing them", async () => { + const p = await project(); + const client = sdk(ADMIN_KEY); + const created = await client.createProjectKey(p.id, { role: "read_only" }); + const { keys } = await client.listProjectKeys(p.id); + expect(keys).toHaveLength(2); + expect(keys.find((k) => k.id === created.id)!.role).toBe("read_only"); + expect(JSON.stringify(keys)).not.toContain(p.apiKey.slice(8)); + await client.deleteProjectKey(p.id, created.id); + expect((await client.listProjectKeys(p.id)).keys).toHaveLength(1); + }); +}); + +// ─── Templates ────────────────────────────────────────────────────────────── + +describe("templates", () => { + it("syncs, lists with filters, reads and deletes templates", async () => { + const p = await project(); + expect( + await p.client.syncTemplates([ + { id: "a", channel: "email", topic: "news", content: { subject: "A" } }, + { id: "b", channel: "sms", content: { text: "B" } }, + ]), + ).toEqual({ synced: 2 }); + expect((await p.api("PUT", "/v1/templates", { templates: [] })).status).toBe(400); + + expect((await p.client.listTemplates()).templates.map((t) => t.id).sort()).toEqual(["a", "b"]); + expect( + (await p.api("GET", "/v1/templates?channel=sms")).body.templates.map((t: any) => t.id), + ).toEqual(["b"]); + expect( + (await p.api("GET", "/v1/templates?topic=news")).body.templates.map((t: any) => t.id), + ).toEqual(["a"]); + expect((await p.api("GET", "/v1/templates?limit=1")).body.templates).toHaveLength(1); + expect(await p.client.getTemplate("a")).toMatchObject({ + id: "a", + topics: ["news"], + content: { subject: "A" }, + }); + + await p.client.deleteTemplate("a"); + await expect(p.client.getTemplate("a")).rejects.toThrow(/template_not_found/); + await expect(p.client.deleteTemplate("a")).rejects.toThrow(/template_not_found/); + }); + + it("uses an edited template for the very next send", async () => { + const p = await project(); + await p.client.addUser({ id: "u", email: "u@x.com" }); + await p.client.syncTemplates([{ id: "t", channel: "email", content: { subject: "v1" } }]); + await p.client.notify({ user: "u", template: "t" }); + await waitFor("v1 sent", () => email.for(p.id).length === 1); + + await p.client.syncTemplates([{ id: "t", channel: "email", content: { subject: "v2" } }]); + await settle(200); + await p.client.notify({ user: "u", template: "t" }); + await waitFor("v2 sent", () => email.for(p.id).length === 2); + expect(email.for(p.id)[1]!.renderedContent.content).toEqual({ subject: "v2" }); + }); +}); + +// ─── Users, contacts, preferences ─────────────────────────────────────────── + +describe("users", () => { + it("creates a user with contacts through every input style, and reads it back", async () => { + const p = await project(); + await p.client.addUser("u1", [{ channel: "sms", target: "+1" }], { + language: "fr", + email: "u1@x.com", + }); + await p.client.identify({ + id: "u2", + phone: ["+2", "+3"], + pushToken: "tok", + segments: ["beta"], + }); + + const u1 = await p.client.getUser("u1"); + expect(u1).toMatchObject({ userId: "u1", language: "fr", email: "u1@x.com" }); + expect(u1.contacts!.map((c) => `${c.channel}:${c.target}`).sort()).toEqual([ + "email:u1@x.com", + "sms:+1", + ]); + + const u2 = await p.client.getUserContacts("u2"); + expect(u2.contacts.map((c) => c.target).sort()).toEqual(["+2", "+3", "tok"]); + expect((await p.client.getUser("u2")).segments).toEqual(["beta"]); + expect((await p.api("POST", "/v1/users", { email: "no-id@x.com" })).status).toBe(400); + }); + + it("updates a user, adding contacts and keeping the old ones", async () => { + const p = await project(); + await p.client.addUser({ id: "u", email: "old@x.com", timezone: "UTC" }); + await p.client.updateUser("u", { + timezone: "Asia/Tokyo", + contacts: [{ channel: "sms", target: "+9", preferences: { topics: { promo: false } } }], + }); + + const u = await p.client.getUser("u"); + expect(u.timezone).toBe("Asia/Tokyo"); + expect(u.email).toBe("old@x.com"); + expect(u.contacts!.map((c) => c.target).sort()).toEqual(["+9", "old@x.com"]); + expect(u.contacts!.find((c) => c.target === "+9")!.preferences).toEqual({ + topics: { promo: false }, + }); + await expect(p.client.updateUser("ghost", { language: "x" })).rejects.toThrow(/user_not_found/); + }); + + it("adds and removes contacts individually and in batches", async () => { + const p = await project(); + await p.client.addUser({ id: "u" }); + await p.client.addContact("u", { channel: "email", target: "a@x.com" }); + await p.client.addContacts("u", [ + { channel: "sms", target: "+1" }, + { channel: "webhook", target: "https://hooks.example.com/a b" }, + ]); + expect((await p.client.getUserContacts("u")).contacts).toHaveLength(3); + + await p.client.deleteContact("u", "webhook", "https://hooks.example.com/a b"); + await p.client.deleteContact("u", "sms", "+1"); + expect((await p.client.getUserContacts("u")).contacts.map((c) => c.target)).toEqual([ + "a@x.com", + ]); + + await expect(p.client.deleteContact("u", "sms", "+1")).rejects.toThrow(/contact_not_found/); + expect((await p.api("DELETE", "/v1/users/u/contacts/pigeon/x")).status).toBe(400); + expect( + (await p.api("POST", "/v1/users/ghost/contacts", { channel: "sms", target: "+1" })).status, + ).toBe(404); + expect((await p.api("POST", "/v1/users/u/contacts", { channel: "sms" })).status).toBe(400); + }); + + it("reads and updates preferences", async () => { + const p = await project(); + await p.client.addUser({ id: "u", preferences: { topics: { news: true } } }); + await p.client.updateUserPreferences("u", { + channels: { sms: false }, + quietHours: [{ start: "22:00", end: "07:00" }], + }); + expect(await p.client.getUserPreferences("u")).toEqual({ + topics: { news: true }, + channels: { sms: false }, + quietHours: [{ start: "22:00", end: "07:00" }], + }); + expect( + ( + await p.api("PATCH", "/v1/users/u/preferences", { + quietHours: [{ start: "25:00", end: "x" }], + }) + ).status, + ).toBe(400); + expect((await p.api("GET", "/v1/users/ghost/preferences")).status).toBe(404); + expect((await p.api("PATCH", "/v1/users/ghost/preferences", {})).status).toBe(404); + }); + + it("deletes a user", async () => { + const p = await project(); + await p.client.addUser({ id: "u", email: "u@x.com" }); + await p.client.deleteUser("u"); + await expect(p.client.getUser("u")).rejects.toThrow(/user_not_found/); + await expect(p.client.deleteUser("u")).rejects.toThrow(/user_not_found/); + }); + + it("lists and filters users, paging with the cursor", async () => { + const p = await project(); + for (let i = 0; i < 7; i++) { + await p.client.addUser({ + id: `user-${i}`, + email: `user${i}@x.com`, + language: i % 2 ? "de" : "en", + segments: i < 2 ? ["vip"] : [], + }); + } + const seen: string[] = []; + let cursor: string | undefined; + for (;;) { + const page = await p.client.listUsers({ limit: 3, cursor }); + seen.push(...page.users.map((u) => u.userId)); + if (!page.nextCursor) break; + cursor = page.nextCursor; + } + expect(seen.sort()).toEqual(Array.from({ length: 7 }, (_, i) => `user-${i}`)); + expect((await p.client.listUsers({ language: "de" })).users).toHaveLength(3); + expect((await p.client.listUsers({ segment: "vip" })).users).toHaveLength(2); + expect((await p.client.listUsers({ search: "user6" })).users.map((u) => u.userId)).toEqual([ + "user-6", + ]); + expect((await p.client.listSegments()).segments).toEqual(["vip"]); + }); + + it("shows a user's recent messages in their details", async () => { + const p = await project(); + await p.client.syncTemplates([{ id: "t", channel: "email", content: { subject: "s" } }]); + await p.client.addUser({ id: "u", email: "detail@x.com" }); + await p.client.notify({ user: "u", template: "t" }); + let details: any; + await waitFor("logged", async () => { + details = await p.client.getUserDetails("u"); + return details.logs.some((l: any) => l.status === "delivered"); + }); + expect(details.contacts.map((c: any) => c.target)).toEqual(["detail@x.com"]); + }); +}); + +// ─── Notify ───────────────────────────────────────────────────────────────── + +describe("notify", () => { + it("validates the request", async () => { + const p = await project(); + expect((await p.api("POST", "/v1/notify", { template: "t" })).status).toBe(400); + expect( + (await p.api("POST", "/v1/notify", { user: "u", segment: "s", template: "t" })).status, + ).toBe(400); + expect((await p.api("POST", "/v1/notify", { user: "u" })).status).toBe(400); + expect( + (await p.api("POST", "/v1/notify", "{not json", { "content-type": "application/json" })) + .status, + ).toBe(400); + }); + + it("fans a list of users out into one request each", async () => { + const p = await project(); + await p.client.syncTemplates([{ id: "t", channel: "email", content: { subject: "s" } }]); + for (const id of ["a", "b", "c"]) await p.client.addUser({ id, email: `${id}@x.com` }); + const res = await p.api("POST", "/v1/notify", { + user: ["a", "b", "c"], + template: "t", + campaign: "trio", + }); + expect(res.status).toBe(202); + expect(res.body.messageIds).toHaveLength(3); + expect(res.body.campaign).toBe("trio"); + await waitFor("3 sent", () => email.for(p.id).length === 3); + }); + + it("refuses a payload larger than 5MB", async () => { + const p = await project(); + const res = await p + .api("POST", "/v1/notify", { + user: "u", + template: "t", + data: { blob: "x".repeat(6 * 1024 * 1024) }, + }) + .catch((err) => ({ status: 413, err })); + expect(res.status).toBe(413); + }); +}); + +// ─── Scheduled notifications ──────────────────────────────────────────────── + +describe("scheduled notifications", () => { + it("lists them page by page and cancels one, only within its own project", async () => { + const p = await project(); + const other = await project(); + await p.client.syncTemplates([{ id: "t", channel: "email", content: { subject: "s" } }]); + for (const id of ["a", "b", "c"]) await p.client.addUser({ id, email: `${id}@x.com` }); + const sendAt = new Date(Date.now() + 3600_000).toISOString(); + await p.api("POST", "/v1/notify", { user: ["a", "b", "c"], template: "t", sendAt }); + + await waitFor( + "3 parked", + async () => (await p.client.getScheduledMessages()).scheduled.length === 3, + ); + const first = await p.api("GET", "/v1/notifications/scheduled?limit=2"); + const second = await p.api( + "GET", + `/v1/notifications/scheduled?limit=2&cursor=${encodeURIComponent(first.body.nextCursor)}`, + ); + expect(first.body.scheduled).toHaveLength(2); + expect(second.body.scheduled).toHaveLength(1); + expect(second.body.nextCursor).toBeNull(); + expect((await p.api("GET", "/v1/notifications/scheduled?channel=sms")).body.scheduled).toEqual( + [], + ); + expect((await other.client.getScheduledMessages()).scheduled).toEqual([]); + + const taskId = first.body.scheduled[0].taskId; + await expect(other.client.cancelNotification(taskId)).rejects.toThrow(); + expect(await p.client.cancelNotification(taskId)).toEqual({ success: true }); + await expect(p.client.cancelNotification(taskId)).rejects.toThrow(); + expect((await p.client.getScheduledMessages()).scheduled).toHaveLength(2); + }); + + it("reports an unknown notification as not found", async () => { + const p = await project(); + await expect(p.client.getNotificationStatus("nope")).rejects.toThrow(/not found/i); + }); +}); + +// ─── Suppressions ─────────────────────────────────────────────────────────── + +describe("suppressions", () => { + it("creates (normalised), lists with filters, and deletes suppressions", async () => { + const p = await project(); + expect( + await p.client.createSuppression({ + channel: "email", + target: " Mixed@Case.COM ", + reason: "bounced", + }), + ).toEqual({ + channel: "email", + target: "mixed@case.com", + reason: "bounced", + }); + await p.client.createSuppression({ channel: "sms", target: "+15550001" }); + await p.client.createSuppression({ channel: "sms", target: "+15550001" }); // idempotent + expect((await p.api("POST", "/v1/suppressions", { channel: "fax", target: "x" })).status).toBe( + 400, + ); + + expect((await p.client.listSuppressions()).suppressions).toHaveLength(2); + expect( + (await p.client.listSuppressions({ channel: "sms" })).suppressions.map((s) => s.target), + ).toEqual(["+15550001"]); + expect((await p.client.listSuppressions({ reason: "bounced" })).suppressions).toHaveLength(1); + expect((await p.client.listSuppressions({ target: "MIXED" })).suppressions).toHaveLength(1); + + await p.client.deleteSuppression("email", "MIXED@case.com"); + expect((await p.client.listSuppressions()).suppressions.map((s) => s.channel)).toEqual(["sms"]); + }); +}); + +// ─── Unsubscribe links ────────────────────────────────────────────────────── + +describe("unsubscribe", () => { + beforeEach(clearRateLimits); + function token( + claim: Partial[0]> & { projectId: string }, + secret = SECRET, + ) { + return signUnsubscribeToken( + { userId: "u", channel: "email", target: "u@x.com", topics: ["news"], ...claim }, + secret, + ); + } + const anon = () => apiClient(app.baseUrl, undefined, { "x-forwarded-for": randomUUID() }); + + it("shows a confirmation page on GET without unsubscribing anyone", async () => { + const p = await project(); + await p.client.addUser({ id: "u", email: "u@x.com" }); + const page = await anon()( + "GET", + `/v1/unsubscribe?token=${encodeURIComponent(token({ projectId: p.id }))}`, + ); + expect(page.status).toBe(200); + expect(page.headers.get("content-type")).toMatch(/text\/html/); + expect(page.body).toContain('
{ + const p = await project(); + const forged = token({ projectId: p.id }, "some-other-secret"); + expect( + (await anon()("GET", `/v1/unsubscribe?token=${encodeURIComponent(forged)}`)).status, + ).toBe(400); + expect( + (await anon()("POST", `/v1/unsubscribe?token=${encodeURIComponent(forged)}`)).status, + ).toBe(400); + expect((await anon()("POST", "/v1/unsubscribe")).status).toBe(400); + }); + + it("opts the user out of the template's topics on POST", async () => { + const p = await project(); + await p.client.addUser({ id: "u", email: "u@x.com" }); + const res = await anon()( + "POST", + `/v1/unsubscribe?token=${encodeURIComponent(token({ projectId: p.id }))}`, + "List-Unsubscribe=One-Click", + { + "content-type": "application/x-www-form-urlencoded", + }, + ); + expect(res.status).toBe(200); + expect((await p.client.getUserPreferences("u")).topics).toEqual({ news: false }); + }); + + it("suppresses the address outright when there is no topic to scope to, or the user is gone", async () => { + const p = await project(); + await anon()( + "POST", + `/v1/unsubscribe?token=${encodeURIComponent(token({ projectId: p.id, topics: [], target: "Whole@X.com" }))}`, + ); + await anon()( + "POST", + `/v1/unsubscribe?token=${encodeURIComponent(token({ projectId: p.id, userId: "deleted", target: "gone@x.com" }))}`, + ); + const targets = (await p.client.listSuppressions()).suppressions.map((s) => s.target).sort(); + expect(targets).toEqual(["gone@x.com", "whole@x.com"]); + }); + + it("caps unsubscribe requests per client address", async () => { + await clearRateLimits(); + const same = apiClient(app.baseUrl, undefined, { "x-forwarded-for": "203.0.113.9" }); + const statuses: number[] = []; + for (let i = 0; i < 62; i++) + statuses.push((await same("GET", "/v1/unsubscribe?token=x")).status); + // Without TRUST_PROXY the header is ignored and every request is from the + // loopback address, which is what is being limited here. + expect(statuses.slice(0, 60).every((s) => s === 400)).toBe(true); + expect(statuses.slice(60)).toEqual([429, 429]); + await clearRateLimits(); + }); +}); + +// ─── Provider webhooks ────────────────────────────────────────────────────── + +describe("provider webhooks", () => { + const hook = (body: unknown, signature = "valid") => + apiClient(app.baseUrl)("POST", "/webhooks/test", body, { "x-signature": signature }); + + it("answers the provider's verification challenge", async () => { + const ok = await fetch(`${app.baseUrl}/webhooks/test?hub.verify_token=ok&hub.challenge=abc123`); + expect(ok.status).toBe(200); + expect(await ok.text()).toBe("abc123"); + expect((await fetch(`${app.baseUrl}/webhooks/test?hub.verify_token=no`)).status).toBe(403); + }); + + it("rejects an unsigned webhook", async () => { + expect((await hook({ events: [] }, "forged")).status).toBe(401); + }); + + it("records engagement against the message and suppresses dead or unwilling addresses", async () => { + const p = await project(); + await p.client.syncTemplates([{ id: "t", channel: "email", content: { subject: "s" } }]); + for (const id of ["hard", "soft", "unsub", "open"]) + await p.client.addUser({ id, email: `${id}@x.com` }); + for (const id of ["hard", "soft", "unsub", "open"]) + await p.client.notify({ user: id, template: "t", campaign: "hooks" }); + await waitFor("4 sent", () => email.for(p.id).length === 4); + await waitFor("4 logged", async () => { + const rows = + await infra.sql`SELECT 1 FROM message_logs WHERE project_id = ${p.id} AND status = 'delivered'`; + return rows.length === 4; + }); + const pm = (id: string) => `pm-${email.for(p.id).find((t) => t.recipientId === id)!.taskId}`; + + const events: WebhookEvent[] = [ + { + providerMessageId: pm("hard"), + status: "bounced", + bounceType: "hard", + recipient: "HARD@x.com", + }, + { + providerMessageId: pm("soft"), + status: "bounced", + bounceType: "soft", + recipient: "soft@x.com", + }, + { providerMessageId: pm("unsub"), status: "unsubscribed", recipient: "unsub@x.com" }, + { providerMessageId: pm("open"), status: "opened" }, + { providerMessageId: pm("open"), status: "opened" }, + { + providerMessageId: "pm-from-someone-else", + status: "complained", + recipient: "victim@x.com", + }, + ]; + expect((await hook({ events })).status).toBe(200); + expect((await hook({ events })).status).toBe(200); // providers retry; must be harmless + + const targets = (await p.client.listSuppressions()).suppressions + .map((s) => `${s.target}:${s.reason}`) + .sort(); + expect(targets).toEqual(["hard@x.com:bounced", "unsub@x.com:unsubscribed"]); + const [{ n }] = + (await infra.sql`SELECT count(*)::int AS n FROM suppressions WHERE target = 'victim@x.com'`) as any; + expect(n).toBe(0); + + const stats = await p.client.getCampaignStats("hooks"); + expect(stats.totals).toMatchObject({ + sent: 4, + delivered: 4, + opened: 1, + bounced: 2, + unsubscribed: 1, + }); + expect(stats.engagementTracked).toBe(true); + }); +}); + +// ─── Events, campaigns, system ────────────────────────────────────────────── + +describe("events", () => { + it("accepts an event onto the events stream", async () => { + const p = await project(); + const res = await p.client.ingestEvent({ name: "order.paid", properties: { orderId: "o1" } }); + expect(res.eventId).toMatch(/^[0-9a-f-]{36}$/); + const entries = await infra.redis.xrange(STREAMS.EVENTS_INBOUND, "-", "+"); + const ev = entries + .map(([, f]) => JSON.parse(f[1]!)) + .find((e) => e.metadata.traceId === res.eventId); + expect(ev.payload).toEqual({ + projectId: p.id, + eventName: "order.paid", + payload: { orderId: "o1" }, + }); + }); + + it("refuses an expired event and a malformed one", async () => { + const p = await project(); + const old = await p.api( + "POST", + "/v1/events", + { name: "e", properties: {} }, + { + "x-timestamp": new Date(Date.now() - 600_000).toISOString(), + "x-expiry": "60", + }, + ); + expect(old.status).toBe(400); + expect((await p.api("POST", "/v1/events", { name: "e" })).status).toBe(400); + }); + + it("streams this project's deliveries, and only this project's, over SSE", async () => { + const p = await project(); + const other = await project(); + for (const x of [p, other]) { + await x.client.syncTemplates([{ id: "t", channel: "email", content: { subject: "s" } }]); + await x.client.addUser({ id: "u", email: "u@x.com" }); + } + + const controller = new AbortController(); + const res = await fetch(`${app.baseUrl}/v1/events/stream`, { + headers: { authorization: `Bearer ${p.apiKey}` }, + signal: controller.signal, + }); + expect(res.headers.get("content-type")).toBe("text/event-stream"); + const reader = res.body!.getReader(); + let text = ""; + const reading = (async () => { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + text += new TextDecoder().decode(value); + } + })().catch(() => {}); + + await other.client.notify({ user: "u", template: "t" }); + await p.client.notify({ user: "u", template: "t" }); + await waitFor("own delivery streamed", () => text.includes("event: delivery:delivered")); + await settle(500); + controller.abort(); + await reading; + + const taskIds = [...text.matchAll(/"taskId":"([^"]+)"/g)].map((m) => m[1]); + const mine = email.for(p.id).map((t) => t.taskId); + expect(taskIds.length).toBeGreaterThan(0); + expect(taskIds.every((id) => mine.includes(id!))).toBe(true); + }); +}); + +describe("campaigns", () => { + it("lists campaigns with their filters, and 404s an unknown one", async () => { + const p = await project(); + await p.client.syncTemplates([{ id: "t", channel: "email", content: { subject: "s" } }]); + for (const id of ["a", "b"]) await p.client.addUser({ id, email: `${id}@x.com` }); + await p.client.notify({ user: "a", template: "t", campaign: "alpha" }); + await p.client.notify({ user: ["a", "b"], template: "t", campaign: "beta" } as any); + await waitFor("logged", async () => (await p.client.listCampaigns()).campaigns.length === 2); + await waitFor( + "counted", + async () => (await p.client.listCampaigns({ search: "bet" })).campaigns[0]?.messages === 2, + ); + + expect( + (await p.client.listCampaigns({ minMessages: 2 })).campaigns.map((c) => c.campaign), + ).toEqual(["beta"]); + expect((await p.client.listCampaigns({ channel: "sms" })).campaigns).toEqual([]); + expect( + (await p.client.listCampaigns({ since: new Date(Date.now() + 3600_000) })).campaigns, + ).toEqual([]); + expect( + (await p.client.listCampaigns({ until: new Date(Date.now() - 3600_000) })).campaigns, + ).toEqual([]); + await expect(p.client.getCampaignStats("nope")).rejects.toThrow(/No messages/); + }); +}); + +describe("system", () => { + it("reports health, readiness and liveness", async () => { + expect((await fetch(`${app.baseUrl}/live`)).status).toBe(200); + expect((await fetch(`${app.baseUrl}/ready`)).status).toBe(200); + await waitFor("workers reported", async () => { + const h: any = await (await fetch(`${app.baseUrl}/health`)).json(); + return h.workers.delivery.state === "running"; + }); + const health: any = await (await fetch(`${app.baseUrl}/health`)).json(); + expect(health).toMatchObject({ status: "ok", redis: true, database: true }); + expect(health.workers.ai.status).toBe("unknown"); + }); + + it("serves Prometheus metrics to loopback callers", async () => { + const res = await fetch(`${app.baseUrl}/metrics`); + expect(res.status).toBe(200); + expect(await res.text()).toContain("notifkit_messages_processed_total"); + }); + + it("reports system health and per-project delivery stats", async () => { + const p = await project(); + await p.client.syncTemplates([{ id: "t", channel: "email", content: { subject: "s" } }]); + await p.client.addUser({ id: "u", email: "u@x.com" }); + await p.client.notify({ user: "u", template: "t" }); + await waitFor( + "logged", + async () => (await p.client.getSystemMetrics()).deliveryStats.delivered === 1, + ); + + const metrics = await p.client.getSystemMetrics(); + expect(metrics.deliveryStats).toEqual({ total: 1, delivered: 1, failed: 0, successRate: 100 }); + const health = await sdk(ADMIN_KEY).getSystemHealth(); + expect(health.status).toBe("healthy"); + expect(Object.keys(health.workers)).toContain("engine"); + }); + + it("deletes a dead-lettered message, but only the project's own", async () => { + const p = await project(); + const other = await project(); + const ev = { + id: randomUUID(), + type: "notification.dispatched", + timestamp: new Date().toISOString(), + payload: { projectId: p.id }, + metadata: { traceId: "t", source: "s", retryCount: 0 }, + dlq: { + originalStream: STREAMS.OUTBOUND_NORMAL, + ackedAt: new Date().toISOString(), + reason: "boom", + }, + }; + const id = await infra.redis.xadd(STREAMS.DEAD_LETTER, "*", "data", JSON.stringify(ev)); + + const listed = await p.client.getDLQMessages(); + expect(listed.messages.find((m) => m.id === id)).toMatchObject({ + eventType: "notification.dispatched", + error: "boom", + }); + expect((await other.client.getDLQMessages()).messages.find((m) => m.id === id)).toBeUndefined(); + await expect(other.client.deleteDLQMessage(id!)).rejects.toThrow(); + await expect(other.client.replayDLQMessage(id!)).rejects.toThrow(); + + expect(await p.client.deleteDLQMessage(id!)).toEqual({ success: true }); + expect(await infra.redis.xlen(STREAMS.DEAD_LETTER)).toBe(0); + await expect(p.client.replayDLQMessage("")).rejects.toThrow(); + }); +}); + +describe("http plumbing", () => { + it("answers CORS preflights, echoing only the configured origin on admin routes", async () => { + const pre = await fetch(`${app.baseUrl}/v1/notify`, { method: "OPTIONS" }); + expect(pre.status).toBe(204); + expect(pre.headers.get("access-control-allow-origin")).toBe("*"); + + const allowed = await fetch(`${app.baseUrl}/v1/projects`, { + method: "OPTIONS", + headers: { origin: "https://dash.example.com" }, + }); + expect(allowed.headers.get("access-control-allow-origin")).toBe("https://dash.example.com"); + const denied = await fetch(`${app.baseUrl}/v1/projects`, { + method: "OPTIONS", + headers: { origin: "https://evil.example.com" }, + }); + expect(denied.headers.get("access-control-allow-origin")).toBeNull(); + }); + + it("404s an unknown route", async () => { + const p = await project(); + expect((await p.api("GET", "/v1/nothing-here")).status).toBe(404); + }); + + it("the SDK client surfaces a non-JSON error response as an error with its status", async () => { + await clearRateLimits(); + const client = new NotifkitClient({ baseUrl: app.baseUrl }); + await expect((client as any).request("/v1/unsubscribe?token=bad", "POST")).rejects.toThrow( + /400/, + ); + }); +}); + +describe("SDK client helpers", () => { + it("sync() pushes the templates given at construction, and is a no-op without any", async () => { + const p = await project(); + const withTemplates = new NotifkitClient({ + baseUrl: app.baseUrl, + apiKey: p.apiKey, + templates: [{ id: "x", channel: "email", content: {} }], + }); + expect(await withTemplates.sync()).toEqual({ synced: 1 }); + expect(await p.client.sync()).toEqual({ synced: 0 }); + expect((await p.client.listTemplates()).templates.map((t) => t.id)).toEqual(["x"]); + }); + + it("covers workflows through the client", async () => { + const p = await project(); + await p.client.createWorkflow({ name: "wf-a", steps: [{ action: "wait", duration: "1h" }] }); + await p.client.createWorkflow({ name: "wf-b", steps: [{ action: "wait", duration: "1h" }] }); + expect((await p.client.listWorkflows({ search: "A" })).workflows.map((w) => w.name)).toEqual([ + "wf-a", + ]); + expect((await p.client.listWorkflows({ limit: 1 })).workflows).toHaveLength(1); + const trig = await p.client.triggerWorkflow({ + name: "wf-a", + user: { id: "inline-user", email: "i@x.com" }, + }); + expect(trig.instanceId).toMatch(/^[0-9a-f-]{36}$/); + await expect(p.client.getWorkflow(randomUUID())).rejects.toThrow(/workflow_not_found/); + await expect(p.client.getWorkflow("not-a-uuid")).rejects.toThrow(/workflow_not_found/); + await expect(p.client.cancelWorkflow(randomUUID())).rejects.toThrow(/workflow_not_cancelable/); + // The inline user was created by the trigger. + await waitFor( + "inline user", + async () => (await p.api("GET", "/v1/users/inline-user")).status === 200, + ); + }); + + it("reads the delivery log with filters", async () => { + const p = await project(); + await p.client.syncTemplates([ + { id: "t1", channel: "email", content: { subject: "s" } }, + { id: "t2", channel: "email", content: { subject: "s" } }, + ]); + await p.client.addUser({ id: "u", email: "u@x.com" }); + await p.client.notify({ user: "u", template: "t1", campaign: "c1" }); + await p.client.notify({ user: "u", template: "t2" }); + await waitFor( + "both logged", + async () => (await p.client.getNotificationLogs({ status: "delivered" })).logs.length === 2, + ); + + expect( + (await p.client.getNotificationLogs({ templateId: "t1" })).logs.map((l) => l.templateId), + ).toEqual(["t1"]); + expect((await p.client.getNotificationLogs({ campaign: "c1" })).logs).toHaveLength(1); + expect((await p.client.getNotificationLogs({ channel: "sms" })).logs).toEqual([]); + const one = (await p.client.getNotificationLogs({ templateId: "t2" })).logs[0]!; + expect((await p.client.getNotificationLogs({ taskId: one.taskId })).logs).toHaveLength(1); + expect( + (await p.client.getNotificationLogs({ search: one.taskId.slice(0, 12) })).logs.length, + ).toBeGreaterThanOrEqual(1); + expect((await p.client.getNotificationStatus(one.taskId)).status).toBe("delivered"); + }); +}); + +describe("admin dashboard", () => { + // Pointed at a port nothing listens on, so the static bundle is served. + let restore: string | undefined; + beforeAll(async () => { + restore = process.env.VITE_DEV_URL; + const { freePort } = await import("./support/pipeline.js"); + process.env.VITE_DEV_URL = `http://127.0.0.1:${await freePort()}`; + }); + afterAll(() => { + if (restore === undefined) delete process.env.VITE_DEV_URL; + else process.env.VITE_DEV_URL = restore; + }); + + const get = (path: string) => fetch(`${app.baseUrl}${path}`, { redirect: "manual" }); + + it("redirects /admin to /admin/ and serves the SPA's index", async () => { + const redirect = await get("/admin?x=1"); + expect(redirect.status).toBe(301); + expect(redirect.headers.get("location")).toBe("/admin/?x=1"); + + const index = await get("/admin/"); + expect(index.status).toBe(200); + expect(index.headers.get("content-type")).toBe("text/html; charset=utf-8"); + expect(index.headers.get("cache-control")).toContain("must-revalidate"); + expect(await index.text()).toContain('
{ + const { readdirSync } = await import("node:fs"); + const js = readdirSync("dashboard/dist/assets").find((f) => f.endsWith(".js"))!; + const res = await get(`/admin/assets/${js}`); + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toBe("application/javascript; charset=utf-8"); + expect(res.headers.get("cache-control")).toContain("immutable"); + expect((await get("/admin/favicon.svg")).headers.get("content-type")).toBe("image/svg+xml"); + }); + + it("falls back to the index for client-side routes", async () => { + const res = await get("/admin/projects/123/logs"); + expect(res.status).toBe(200); + expect(await res.text()).toContain('
{ + for (const path of [ + "/admin/../package.json", + "/admin/%2e%2e/%2e%2e/package.json", + "/admin/..%2f..%2fpackage.json", + "/admin/..%5c..%5cpackage.json", + ]) { + const res = await get(path); + const body = await res.text(); + expect(body).not.toContain('"name": "notifkit"'); + } + }); + + it("proxies to the dashboard dev server when one is running", async () => { + const http = await import("node:http"); + const dev = http.createServer((req, res) => { + res.writeHead(200, { "content-type": "text/plain", "x-dev": "1" }); + res.end(`dev:${req.method} ${req.url}`); + }); + await new Promise((r) => dev.listen(0, "127.0.0.1", r)); + const before = process.env.VITE_DEV_URL; + process.env.VITE_DEV_URL = `http://127.0.0.1:${(dev.address() as any).port}`; + try { + const res = await get("/admin/src/main.tsx"); + expect(res.headers.get("x-dev")).toBe("1"); + expect(await res.text()).toBe("dev:GET /admin/src/main.tsx"); + } finally { + process.env.VITE_DEV_URL = before; + await new Promise((r) => dev.close(r)); + } + }); +}); diff --git a/tests/integration/delivery-worker.test.ts b/tests/integration/delivery-worker.test.ts new file mode 100644 index 0000000..00fce7b --- /dev/null +++ b/tests/integration/delivery-worker.test.ts @@ -0,0 +1,462 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { randomUUID } from "node:crypto"; +import { DeliveryWorker } from "@/services/delivery/main.js"; +import { StreamConsumer, PendingMessageScanner, StreamProducer } from "@/queue/index.js"; +import { IdempotencyGuard } from "@/idempotency/index.js"; +import { ContactRepository, TemplateRepository, UserRepository } from "@/repositories/index.js"; +import { TemplateCache } from "@/templates/index.js"; +import { transportRegistry, type Transport, type DeliveryResult } from "@/transport/index.js"; +import { STREAMS, CONSUMER_GROUPS, OUTBOUND_STREAMS, buildStreamEvent } from "@/contracts/index.js"; +import { createPriorityProducers } from "@/workers/bootstrap.js"; +import { globalEmitter } from "@/shared/index.js"; +import { createLogger } from "@/logger/index.js"; +import { useInfra, waitFor, settle } from "./support/infra.js"; + +/** + * The delivery worker on its own: real streams, real idempotency leases, + * real Postgres for the outbox and contacts, and transports that behave like + * providers do when they are slow, down, or rejecting. + */ + +const infra = useInfra(); +const logger = createLogger({ name: "test", level: "silent" }); +let project: string; +const workers: DeliveryWorker[] = []; + +beforeEach(async () => { + await infra.reset(); + project = randomUUID(); + await infra.sql`INSERT INTO projects (id, name) VALUES (${project}, 'p')`; +}); + +afterEach(async () => { + await Promise.all(workers.splice(0).map((w) => w.stop())); +}); + +class FakeProvider implements Transport { + calls = 0; + constructor( + readonly channel: any, + private readonly behaviour: (n: number) => Promise | DeliveryResult, + readonly limits?: { limit: number; windowSeconds: number }, + ) {} + async send(): Promise { + return this.behaviour(++this.calls); + } +} + +function registry(...transports: Transport[]) { + const r = new (transportRegistry.constructor as any)(); + transports.forEach((t, i) => r.register(t, transports.length - i)); + return r; +} + +function worker(reg: any, name = "d1") { + const consumer = new StreamConsumer({ + redis: infra.redis, + stream: [...OUTBOUND_STREAMS], + group: CONSUMER_GROUPS.DELIVERY, + consumer: name, + dlqStream: STREAMS.DEAD_LETTER, + blockMs: 100, + logger, + }); + const pendingScanner = new PendingMessageScanner({ + redis: infra.redis, + stream: [...OUTBOUND_STREAMS], + group: CONSUMER_GROUPS.DELIVERY, + consumer: name, + logger, + }); + const w = new DeliveryWorker({ + consumer, + pendingScanner, + logger, + concurrency: 10, + transportRegistry: reg, + idempotency: new IdempotencyGuard({ + redis: infra.redis, + keyPrefix: "notif:processed:delivery", + }), + redis: infra.redis, + scheduledProducer: new StreamProducer({ + redis: infra.redis, + stream: STREAMS.SCHEDULED, + logger, + maxLen: 1000, + }), + enrichedProducers: createPriorityProducers(infra.redis, logger, "ENRICHED"), + contactRepo: new ContactRepository(infra.db), + eventsProducer: new StreamProducer({ + redis: infra.redis, + stream: STREAMS.EVENTS_INBOUND, + logger, + maxLen: 1000, + }), + globalEmitter, + db: infra.db, + templateCache: new TemplateCache(new TemplateRepository(infra.db)), + }); + workers.push(w); + return w; +} + +function dispatched(overrides: Record = {}) { + return { + ...buildStreamEvent( + "notification.dispatched", + { + projectId: project, + taskId: `task-${randomUUID()}`, + enrichedEventId: randomUUID(), + recipientId: "u", + channel: "email", + priority: "normal", + templateVariables: {}, + renderedContent: { content: { subject: "s" } }, + destination: "u@x.com", + recipient: { + id: "u", + email: "u@x.com", + locale: "en", + timezone: "UTC", + preferences: { optedOut: false, channels: [] }, + }, + deliveryOptions: { maxAttempts: 3, timeoutMs: 500 }, + ...overrides, + }, + "engine", + ), + id: randomUUID(), + timestamp: new Date().toISOString(), + }; +} + +async function publish(ev: any) { + await infra.redis.xadd(STREAMS.OUTBOUND_NORMAL, "*", "data", JSON.stringify(ev)); +} + +const events = async () => + (await infra.redis.xrange(STREAMS.EVENTS_INBOUND, "-", "+")).map(([, f]) => JSON.parse(f[1]!)); +const parked = async () => + (await infra.sql`SELECT task_id, payload FROM scheduled_payloads ORDER BY task_id`).map( + (r) => r.payload as any, + ); +const dlq = async () => + (await infra.redis.xrange(STREAMS.DEAD_LETTER, "-", "+")).map(([, f]) => JSON.parse(f[1]!)); + +describe("DeliveryWorker", () => { + it("parks a send the provider timed out on for a later attempt, instead of failing it", async () => { + const hang = new FakeProvider("email", () => new Promise(() => {})); + const w = worker(registry(hang)); + const ev = dispatched(); + await w.processInline(ev as any); + await w.stop(); + + expect(await dlq()).toEqual([]); + expect((await events()).some((e) => e.type === "notification.failed")).toBe(false); + const [retry] = await parked(); + expect(retry).toMatchObject({ taskId: ev.payload.taskId, deliveryAttemptCount: 1 }); + // Backed off by at least the circuit breaker's reset time. + expect(Date.parse(retry.scheduledAt)).toBeGreaterThanOrEqual(Date.now() + 25_000); + // Parked as it arrived: no rendered body or abort signal carried along. + expect(retry.signal).toBeUndefined(); + expect(await infra.redis.xlen(STREAMS.SCHEDULED)).toBe(1); + // The attempt is still in the log, as dispatched. + expect((await events()).filter((e) => e.type === "notification.dispatched")).toHaveLength(1); + // Released, so the re-queued attempt can take the lease. + expect(await infra.redis.exists(`notif:processed:delivery:${ev.payload.taskId}`)).toBe(0); + }); + + it("delivers on a later attempt once the provider is back", async () => { + const flaky = new FakeProvider("email", (n) => + n === 1 + ? { success: false, error: "503 Service Unavailable" } + : { success: true, providerMessageId: "late-1" }, + ); + const w = worker(registry(flaky)); + await w.processInline(dispatched() as any); + + // What the scheduler would release when the retry falls due. + const [retry] = await parked(); + await w.processInline({ + ...buildStreamEvent("notification.dispatched", retry, "scheduler"), + id: randomUUID(), + timestamp: new Date().toISOString(), + } as any); + await w.stop(); + + expect(flaky.calls).toBe(2); + const delivered = (await events()).find((e) => e.type === "notification.delivered"); + expect(delivered.payload).toMatchObject({ providerMessageId: "late-1", attempt: 2 }); + expect(await dlq()).toEqual([]); + }); + + it("fails a provider that is still down on the last attempt, and dead-letters the task", async () => { + const hang = new FakeProvider("email", () => new Promise(() => {})); + const w = worker(registry(hang)); + await w.start(); + const ev = dispatched({ deliveryAttemptCount: 2 }); + await publish(ev); + + await waitFor("dead-lettered", async () => (await dlq()).length === 1, 5_000); + expect((await dlq())[0].dlq.reason).toMatch(/timeout/i); + await waitFor("failure event", async () => + (await events()).some((e) => e.type === "notification.failed"), + ); + const failed = (await events()).find((e) => e.type === "notification.failed"); + expect(failed.payload).toMatchObject({ + taskId: ev.payload.taskId, + failureCode: "provider_error", + attempt: 3, + }); + expect(await parked()).toEqual([]); + // Released, so a replay from the DLQ can try again. + expect(await infra.redis.exists(`notif:processed:delivery:${ev.payload.taskId}`)).toBe(0); + }); + + it("fails over to the next provider for the channel, in priority order", async () => { + const primary = new FakeProvider("email", () => ({ + success: false, + error: "503 from primary", + })); + const backup = new FakeProvider("email", () => ({ + success: true, + providerMessageId: "backup-1", + })); + const w = worker(registry(primary, backup)); + await w.start(); + const ev = dispatched(); + await publish(ev); + + await waitFor("delivered", async () => + (await events()).some((e) => e.type === "notification.delivered"), + ); + expect(primary.calls).toBe(1); + expect(backup.calls).toBe(1); + const [outbox] = + await infra.sql`SELECT provider_message_id FROM delivery_outbox WHERE task_id = ${ev.payload.taskId as string}`; + await waitFor( + "outbox", + async () => + (await infra.sql`SELECT 1 FROM delivery_outbox WHERE provider_message_id = 'backup-1'`) + .length === 1, + ); + void outbox; + }); + + it("does not retry a failure the provider says is permanent", async () => { + const reject = new FakeProvider("email", () => ({ + success: false, + error: "invalid recipient", + retryable: false, + })); + await worker(registry(reject)).processInline(dispatched() as any); + expect(reject.calls).toBe(1); + expect(await parked()).toEqual([]); + expect((await dlq())[0].dlq.reason).toBe("invalid recipient"); + }); + + it("stops calling a provider that keeps failing once its circuit opens, and parks what it turned away", async () => { + const down = new FakeProvider("email", () => ({ success: false, error: "down" })); + const w = worker(registry(down)); + // One at a time: sends already in flight when the circuit opens still go out. + for (let i = 0; i < 8; i++) await w.processInline(dispatched() as any); + + expect(down.calls).toBe(5); + // An open circuit is an outage, not a verdict on the message: nothing is + // dead-lettered, and every task waits for another attempt. + expect(await dlq()).toEqual([]); + const retries = await parked(); + expect(retries).toHaveLength(8); + expect(retries.every((r) => r.deliveryAttemptCount === 1)).toBe(true); + }); + + it("keeps a circuit per provider, even for two providers of the same class", async () => { + const down = new FakeProvider("email", () => ({ success: false, error: "down" })); + const backup = new FakeProvider("email", () => ({ success: true, providerMessageId: "b" })); + const w = worker(registry(down, backup)); + for (let i = 0; i < 8; i++) await w.processInline(dispatched() as any); + + // The backup's successes are not the primary's: its circuit opens after + // five failures, and every task still goes out through the backup. + expect(down.calls).toBe(5); + expect(backup.calls).toBe(8); + expect(await dlq()).toEqual([]); + }); + + it("sends a task once when two workers race for it", async () => { + let sends = 0; + const slow = new FakeProvider("email", async () => { + sends++; + await settle(200); + return { success: true, providerMessageId: "p" }; + }); + const reg = registry(slow); + const ev = dispatched(); + await Promise.all([ + worker(reg, "a").processInline(ev as any), + worker(reg, "b").processInline(ev as any), + ]); + await worker(reg, "c").processInline(ev as any); + expect(sends).toBe(1); + }); + + it("defers sends over the provider's limit, and fails them once the retries run out", async () => { + const limited = new FakeProvider("sms", () => ({ success: true, providerMessageId: "x" }), { + limit: 1, + windowSeconds: 60, + }); + const w = worker(registry(limited)); + await w.start(); + const first = dispatched({ channel: "sms", destination: "+1" }); + const second = dispatched({ channel: "sms", destination: "+2" }); + const exhausted = dispatched({ channel: "sms", destination: "+3", throttleAttemptCount: 3 }); + for (const ev of [first, second, exhausted]) await publish(ev); + + await waitFor("one sent", () => limited.calls === 1); + await waitFor("deferred", async () => (await infra.redis.xlen(STREAMS.SCHEDULED)) === 1); + const [row] = await infra.sql`SELECT payload FROM scheduled_payloads`; + expect(row!.payload.throttleAttemptCount).toBe(1); + expect(Date.parse(row!.payload.scheduledAt)).toBeGreaterThan(Date.now()); + + await waitFor("exhausted one failed", async () => + (await events()).some( + (e) => + e.type === "notification.failed" && + e.payload.failureCode === "provider_throttle_exceeded", + ), + ); + expect(limited.calls).toBe(1); + }); + + describe("provider rate limits are per provider, not per channel", () => { + const ok = () => ({ success: true, providerMessageId: "x" }); + + it("sends through the backup once the primary's limit is spent", async () => { + const primary = new FakeProvider("email", ok, { limit: 1, windowSeconds: 60 }); + const backup = new FakeProvider("email", ok, { limit: 5, windowSeconds: 60 }); + const w = worker(registry(primary, backup)); + for (let i = 0; i < 3; i++) await w.processInline(dispatched() as any); + + expect(primary.calls).toBe(1); + expect(backup.calls).toBe(2); + expect(await parked()).toEqual([]); + }); + + it("uses an unlimited backup behind a limited primary", async () => { + const primary = new FakeProvider("email", ok, { limit: 1, windowSeconds: 60 }); + const backup = new FakeProvider("email", ok); + const w = worker(registry(primary, backup)); + for (let i = 0; i < 3; i++) await w.processInline(dispatched() as any); + + expect([primary.calls, backup.calls]).toEqual([1, 2]); + }); + + it("defers a send only once every provider for the channel is at its limit", async () => { + const primary = new FakeProvider("email", ok, { limit: 1, windowSeconds: 60 }); + const backup = new FakeProvider("email", ok, { limit: 1, windowSeconds: 60 }); + const w = worker(registry(primary, backup)); + for (let i = 0; i < 3; i++) await w.processInline(dispatched() as any); + + expect([primary.calls, backup.calls]).toEqual([1, 1]); + const deferred = await parked(); + expect(deferred).toHaveLength(1); + expect(deferred[0].throttleAttemptCount).toBe(1); + }); + }); + + it("dead-letters a task whose template was deleted after it was dispatched", async () => { + const ok = new FakeProvider("email", () => ({ success: true, providerMessageId: "x" })); + const w = worker(registry(ok)); + await w.start(); + await publish(dispatched({ renderedContent: undefined, templateId: "gone" })); + await waitFor("dead-lettered", async () => (await dlq()).length === 1); + expect((await dlq())[0].dlq.reason).toMatch(/template "gone" not found/); + expect(ok.calls).toBe(0); + }); + + it("renders an unrendered task from its template at send time", async () => { + let seen: any; + const capture = new FakeProvider("email", () => ({ success: true, providerMessageId: "x" })); + (capture as any).send = async (task: any) => { + seen = task.renderedContent; + return { success: true, providerMessageId: "x" }; + }; + await new TemplateRepository(infra.db).upsertMany(project, [ + { id: "t", channel: "email", content: { subject: "Hi {{n}}" } }, + ]); + await worker(registry(capture)).processInline( + dispatched({ + renderedContent: undefined, + templateId: "t", + templateVariables: { n: "Bo" }, + }) as any, + ); + expect(seen).toEqual({ content: { subject: "Hi Bo" } }); + }); + + it("with no provider for the channel, falls back along the chain or records the failure", async () => { + const w = worker(registry()); + await w.processInline(dispatched({ channel: "sms", fallbackChain: ["email"] }) as any); + const enriched = await infra.redis.xrange(STREAMS.ENRICHED_NORMAL, "-", "+"); + const next = JSON.parse(enriched[0]![1][1]!).payload; + expect(next.channel).toBe("email"); + expect(next.fallbackChain).toBeUndefined(); + + await w.processInline(dispatched({ channel: "sms" }) as any); + await w.stop(); + const failed = (await events()).find((e) => e.type === "notification.failed"); + expect(failed.payload.failureCode).toBe("no_transport"); + }); + + describe("push", () => { + async function pushUser(token: string) { + await new UserRepository(infra.db).upsertFull(project, { + userId: "u", + segments: [], + preferences: {}, + }); + await new ContactRepository(infra.db).upsert(project, "u", "push", token); + } + + it("deactivates a dead token and falls back to the next channel", async () => { + await pushUser("dead"); + const push = new FakeProvider("push", () => ({ success: false, invalidToken: true })); + await worker(registry(push)).processInline( + dispatched({ channel: "push", destination: "dead", fallbackChain: ["email"] }) as any, + ); + + const [contact] = await new ContactRepository(infra.db).findByUserId(project, "u"); + expect(contact!.active).toBe(false); + expect(await infra.redis.xrange(STREAMS.ENRICHED_NORMAL, "-", "+")).toHaveLength(1); + }); + + it("retries a push that failed, and dead-letters it once out of attempts", async () => { + await pushUser("tok"); + const push = new FakeProvider("push", () => ({ success: false, error: "FCM unavailable" })); + const w = worker(registry(push)); + await w.processInline(dispatched({ channel: "push", destination: "tok" }) as any); + expect(await dlq()).toEqual([]); + expect(await parked()).toHaveLength(1); + + await w.processInline( + dispatched({ channel: "push", destination: "tok", deliveryAttemptCount: 2 }) as any, + ); + expect((await dlq())[0].dlq.reason).toBe("FCM unavailable"); + const [contact] = await new ContactRepository(infra.db).findByUserId(project, "u"); + expect(contact!.active).toBe(true); + }); + + it("delivers a push and records the provider's id", async () => { + const push = new FakeProvider("push", () => ({ success: true, providerMessageId: "fcm-1" })); + const w = worker(registry(push)); + await w.processInline(dispatched({ channel: "push", destination: "tok" }) as any); + await w.stop(); + const delivered = (await events()).find((e) => e.type === "notification.delivered"); + expect(delivered.payload).toMatchObject({ providerMessageId: "fcm-1", channel: "push" }); + // The dispatch entry rides on the outcome rather than being a second event. + expect(delivered.payload.dispatchedAt).toBeDefined(); + }); + }); +}); diff --git a/tests/integration/event-worker.test.ts b/tests/integration/event-worker.test.ts new file mode 100644 index 0000000..ae60c1d --- /dev/null +++ b/tests/integration/event-worker.test.ts @@ -0,0 +1,305 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { randomUUID } from "node:crypto"; +import { EventWorker } from "@/services/events/main.js"; +import { StreamConsumer, PendingMessageScanner, StreamProducer } from "@/queue/index.js"; +import { STREAMS, CONSUMER_GROUPS, buildStreamEvent } from "@/contracts/index.js"; +import { insertMessageLogs, upsertOutboxProviderIds } from "@/db/bulk.js"; +import { createLogger } from "@/logger/index.js"; +import { useInfra, waitFor, settle } from "./support/infra.js"; + +/** + * The events worker against real streams and a real delivery log: outcome + * events become log rows, and inbound events wake the workflows waiting on + * them. + */ + +const infra = useInfra(); +const logger = createLogger({ name: "test", level: "silent" }); +let project: string; +let worker: EventWorker | undefined; + +beforeEach(async () => { + await infra.reset(); + project = randomUUID(); +}); + +afterEach(async () => { + await worker?.stop(); + worker = undefined; +}); + +/** + * The worker is slow to make its first read, as it is on a loaded machine + * running the whole suite. A wait that passes before the worker has read + * anything then fails here every time, rather than once in a while. + */ +const FIRST_READ_DELAY_MS = 300; + +async function start({ concurrency = 10 } = {}) { + const consumer = new StreamConsumer({ + redis: infra.redis, + stream: STREAMS.EVENTS_INBOUND, + group: CONSUMER_GROUPS.EVENTS, + consumer: "e1", + blockMs: 100, + logger, + }); + const read = consumer.readBatch.bind(consumer); + const stop = consumer.stop.bind(consumer); + let stopped = false; + consumer.stop = async () => { + stopped = true; + await stop(); + }; + consumer.readBatch = async function* () { + await settle(FIRST_READ_DELAY_MS); + if (!stopped) yield* read(); + }; + const pendingScanner = new PendingMessageScanner({ + redis: infra.redis, + stream: STREAMS.EVENTS_INBOUND, + group: CONSUMER_GROUPS.EVENTS, + consumer: "e1", + logger, + }); + worker = new EventWorker({ + consumer, + pendingScanner, + logger, + concurrency, + db: infra.db, + workflowProducer: new StreamProducer({ + redis: infra.redis, + stream: STREAMS.WORKFLOW_INBOUND, + logger, + maxLen: 1000, + }), + }); + await worker.start(); + return worker; +} + +async function emit(type: string, payload: Record) { + const ev = { + ...buildStreamEvent(type, payload, "test"), + id: randomUUID(), + timestamp: new Date().toISOString(), + }; + await infra.redis.xadd(STREAMS.EVENTS_INBOUND, "*", "data", JSON.stringify(ev)); + return ev; +} + +/** The events group's XINFO entry: `lag` is entries not yet read, `pending` read but not acked. */ +async function group(): Promise | undefined> { + const groups = (await infra.redis.xinfo("GROUPS", STREAMS.EVENTS_INBOUND)) as unknown[][]; + return groups + .map((flat) => + Object.fromEntries(flat.flatMap((v, i) => (i % 2 ? [] : [[String(v), flat[i + 1]]]))), + ) + .find((g) => g.name === CONSUMER_GROUPS.EVENTS); +} + +/** + * Every entry on the stream has been read by the group and acked. Pending + * alone is not enough: it is also 0 before the worker has read anything. + */ +const drained = async () => { + const g = await group(); + return g?.lag === 0 && g.pending === 0; +}; +const resumes = async () => + (await infra.redis.xrange(STREAMS.WORKFLOW_INBOUND, "-", "+")).map(([, f]) => JSON.parse(f[1]!)); + +describe("EventWorker: delivery log", () => { + it("writes a dispatched row and an outcome row, in that order, once each", async () => { + await start(); + const dispatchedAt = new Date(Date.now() - 5_000).toISOString(); + const outcome = { + projectId: project, + taskId: "t1", + channel: "email", + attempt: 1, + providerMessageId: "pm-1", + dispatchedAt, + templateId: "tpl", + campaignId: "c", + }; + await emit("notification.delivered", outcome); + await emit("notification.delivered", outcome); // a replay + await emit("notification.failed", { + projectId: project, + taskId: "t2", + channel: "sms", + attempt: 2, + }); + await emit("notification.dispatched", { projectId: project, taskId: "t3", channel: "push" }); + + await waitFor("read and acked", drained); + const rows = + await infra.sql`SELECT task_id, kind, status, attempt, provider_message_id, template_id, campaign_id, "timestamp" FROM message_logs ORDER BY task_id, "timestamp"`; + expect(rows.map((r) => [r.task_id, r.kind, r.status, r.attempt])).toEqual([ + ["t1", "dispatched", "dispatched", 1], + ["t1", "attempt", "delivered", 1], + ["t2", "attempt", "failed", 2], + ["t3", "dispatched", "dispatched", 1], + ]); + expect(new Date(rows[0]!.timestamp).toISOString()).toBe(dispatchedAt); + expect(rows[1]).toMatchObject({ + provider_message_id: "pm-1", + template_id: "tpl", + campaign_id: "c", + }); + }); + + it("flushes everything buffered when it stops", async () => { + // Hold rows in the buffer: a flush threshold out of reach (half of 100) + // and no interval flush, so only stopping can write them. + const w = await start({ concurrency: 100 }); + clearInterval((w as any).flushInterval); + for (let i = 0; i < 5; i++) + await emit("notification.delivered", { + projectId: project, + taskId: `s${i}`, + channel: "email", + }); + + await waitFor("all five buffered", () => (w as any).messageLogBuffer.length === 5); + expect(await infra.sql`SELECT 1 FROM message_logs`).toEqual([]); + + await w.stop(); + worker = undefined; + const [{ n }] = (await infra.sql`SELECT count(*)::int AS n FROM message_logs`) as any; + expect(n).toBe(5); + }); + + it("ignores event types it has no use for", async () => { + await start(); + await emit("something.else", { projectId: project }); + await waitFor("read and acked", drained); + expect(await infra.sql`SELECT 1 FROM message_logs`).toEqual([]); + }); +}); + +describe("EventWorker: waking workflows", () => { + async function waitingInstance( + eventName: string, + match: Record, + expiresInMs = 3600_000, + ) { + const [inst] = await infra.sql` + INSERT INTO workflow_instances (project_id, name, status, input) VALUES (${project}, 'wf', 'pending', ${JSON.stringify({ user: { id: "u" } })}::jsonb) RETURNING id`; + const id = inst!.id as string; + await infra.sql`INSERT INTO workflow_steps (project_id, instance_id, step_index, action) VALUES (${project}, ${id}, '0', 'waitForEvent')`; + await infra.sql`INSERT INTO workflow_waiters (project_id, instance_id, event_name, match_criteria, expires_at) + VALUES (${project}, ${id}, ${eventName}, ${JSON.stringify(match)}::jsonb, ${new Date(Date.now() + expiresInMs).toISOString()})`; + return id; + } + + it("resumes the instance whose criteria match, recording the event on its step", async () => { + await start(); + const alice = await waitingInstance("order.paid", { "customer.id": "alice" }); + const bob = await waitingInstance("order.paid", { "customer.id": "bob" }); + + await emit("event.received", { + projectId: project, + eventName: "order.paid", + payload: { customer: { id: "alice" }, total: 5 }, + }); + await waitFor("resumed", async () => (await resumes()).length === 1); + + const [resume] = await resumes(); + expect(resume.type).toBe("workflow.resumed"); + expect(resume.payload).toMatchObject({ + projectId: project, + instanceId: alice, + name: "wf", + input: { user: { id: "u" } }, + reason: "event_matched", + }); + const [step] = await infra.sql`SELECT output FROM workflow_steps WHERE instance_id = ${alice}`; + expect(step!.output).toEqual({ customer: { id: "alice" }, total: 5 }); + const waiters = await infra.sql`SELECT instance_id FROM workflow_waiters`; + expect(waiters.map((w) => w.instance_id)).toEqual([bob]); + }); + + it("does not wake anything for an event in another project, of another name, or after the waiter expired", async () => { + await start(); + await waitingInstance("order.paid", {}); + await waitingInstance("expired", {}, -1_000); + await emit("event.received", { projectId: randomUUID(), eventName: "order.paid", payload: {} }); + await emit("event.received", { projectId: project, eventName: "order.refunded", payload: {} }); + await emit("event.received", { projectId: project, eventName: "expired", payload: {} }); + await emit("event.received", { eventName: "order.paid", payload: {} }); + + await waitFor("read and acked", drained); + await settle(200); + expect(await resumes()).toEqual([]); + }); + + it("drops a waiter whose instance no longer exists", async () => { + await start(); + const id = await waitingInstance("ping", {}); + // Orphan the waiter as a crash between deletes would. + await infra.sql`ALTER TABLE workflow_waiters DISABLE TRIGGER ALL`; + await infra.sql`DELETE FROM workflow_instances WHERE id = ${id}`.catch(() => {}); + await infra.sql`ALTER TABLE workflow_waiters ENABLE TRIGGER ALL`; + const orphaned = (await infra.sql`SELECT 1 FROM workflow_waiters`).length; + + await emit("event.received", { + projectId: project, + eventName: "ping", + payload: "not an object", + }); + await waitFor("read and acked", drained); + expect(await resumes()).toEqual([]); + if (orphaned) expect(await infra.sql`SELECT 1 FROM workflow_waiters`).toEqual([]); + }); +}); + +describe("bulk log writers", () => { + it("insertMessageLogs writes typed rows and skips ones already recorded", async () => { + const row = { + projectId: project, + taskId: "b1", + providerMessageId: null, + channel: "email", + attempt: 1, + kind: "attempt", + status: "delivered", + templateId: null, + workflowInstanceId: null, + campaignId: null, + }; + await insertMessageLogs(infra.db, [ + row, + { + ...row, + taskId: "b2", + workflowInstanceId: randomUUID(), + timestamp: "2020-01-01T00:00:00.000Z", + }, + ]); + await insertMessageLogs(infra.db, [row]); + await insertMessageLogs(infra.db, []); + const rows = await infra.sql`SELECT task_id, "timestamp" FROM message_logs ORDER BY task_id`; + expect(rows.map((r) => r.task_id)).toEqual(["b1", "b2"]); + expect(new Date(rows[1]!.timestamp).toISOString()).toBe("2020-01-01T00:00:00.000Z"); + }); + + it("upsertOutboxProviderIds records and updates provider ids, even with a task twice in one batch", async () => { + const base = { taskId: "o1", channel: "email", destination: "a@x.com" }; + await upsertOutboxProviderIds(infra.db, [{ ...base, providerMessageId: null }]); + await upsertOutboxProviderIds(infra.db, [ + { ...base, providerMessageId: "first" }, + { ...base, providerMessageId: "retry" }, + { taskId: "o2", channel: "sms", destination: "+1", providerMessageId: "s" }, + ]); + await upsertOutboxProviderIds(infra.db, []); + const rows = + await infra.sql`SELECT task_id, provider_message_id FROM delivery_outbox ORDER BY task_id`; + expect(rows.map((r) => [r.task_id, r.provider_message_id])).toEqual([ + ["o1", "retry"], + ["o2", "s"], + ]); + }); +}); diff --git a/tests/integration/mcp.test.ts b/tests/integration/mcp.test.ts new file mode 100644 index 0000000..4c40530 --- /dev/null +++ b/tests/integration/mcp.test.ts @@ -0,0 +1,440 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { randomUUID } from "node:crypto"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { createServer } from "../../packages/mcp/src/index.js"; +import { NotifkitApi, NotifkitApiError } from "../../packages/mcp/src/client.js"; +import { STREAMS } from "@/contracts/index.js"; +import { useInfra, waitFor } from "./support/infra.js"; +import { ADMIN_KEY, RecordingTransport, startNotifkit, type Running } from "./support/pipeline.js"; + +/** + * The MCP server as an agent uses it: a real MCP client, connected over the + * SDK's in-memory transport, calls tools; the tools call a running notifkit. + * Every tool is exercised and judged by its effect, not by the URL it built. + */ + +const infra = useInfra(); +let app: Running; +const email = new RecordingTransport("email"); +const telegram = new RecordingTransport("telegram"); +const slack = new RecordingTransport("slack"); + +beforeAll(async () => { + app = await startNotifkit(infra, { + services: ["api", "enricher", "engine", "delivery", "scheduler", "events", "workflow"], + providers: [email, telegram, slack], + env: { RATE_LIMIT_PER_HOUR: "100000" }, + }); +}, 120_000); + +afterAll(async () => { + await app?.stop(); +}, 60_000); + +async function agent(apiKey: string, projectId?: string) { + const server = createServer({ baseUrl: app.baseUrl, apiKey, projectId }); + const client = new Client({ name: "test-agent", version: "1.0.0" }); + const [a, b] = InMemoryTransport.createLinkedPair(); + await Promise.all([server.connect(a), client.connect(b)]); + + async function call(name: string, args: Record = {}) { + const res: any = await client.callTool({ name, arguments: args }); + const text = res.content[0].text as string; + let body: any = text; + try { + body = JSON.parse(text); + } catch { + // plain text result + } + return { isError: Boolean(res.isError), text, body }; + } + return { client, call, close: () => client.close() }; +} + +async function projectAgent() { + const p = await app.project(); + return { ...p, ...(await agent(p.apiKey)) }; +} + +describe("MCP server", () => { + it("lists all of its tools with input schemas", async () => { + const a = await projectAgent(); + const { tools } = await a.client.listTools(); + expect(tools).toHaveLength(46); + for (const t of tools) expect(t.inputSchema.type).toBe("object"); + await a.close(); + }); + + it("sends to existing users and reports the campaign", async () => { + const a = await projectAgent(); + expect( + ( + await a.call("upsert_template", { + id: "hi", + channel: "email", + content: { subject: "Hi {{name}}" }, + }) + ).isError, + ).toBe(false); + expect( + (await a.call("upsert_user", { id: "u1", email: "u1@x.com", segments: ["beta"] })).isError, + ).toBe(false); + + const sent = await a.call("send_notification", { + template: "hi", + user: "u1", + data: { name: "Ann" }, + campaign: "c-1", + }); + expect(sent.isError).toBe(false); + await waitFor("delivered", () => email.for(a.id).length === 1); + expect(email.for(a.id)[0]!.renderedContent.content).toEqual({ subject: "Hi Ann" }); + + await waitFor( + "stats", + async () => + (await a.call("get_campaign_stats", { campaign: "c-1" })).body?.totals?.delivered === 1, + ); + const campaigns = await a.call("list_campaigns", {}); + expect(campaigns.body.campaigns.map((c: any) => c.campaign)).toEqual(["c-1"]); + await a.close(); + }); + + it("send_campaign reaches every distinct address once, on email and on chat channels", async () => { + const a = await projectAgent(); + await a.call("upsert_template", { + id: "promo", + channel: "email", + content: { subject: "Sale" }, + }); + await a.call("upsert_template", { id: "ping", channel: "telegram", content: { text: "Ping" } }); + await a.call("upsert_template", { id: "note", channel: "slack", content: { text: "Note" } }); + + const res = await a.call("send_campaign", { + campaign: "spring", + template: "promo", + emails: ["A@x.com", "a@x.com", "b@x.com"], + }); + expect(res.body).toMatchObject({ queued: 2, duplicatesRemoved: 1 }); + await waitFor("emails", () => email.for(a.id).length === 2); + expect( + email + .for(a.id) + .map((t) => t.destination) + .sort(), + ).toEqual(["a@x.com", "b@x.com"]); + + await a.call("send_campaign", { + campaign: "chat", + template: "ping", + channel: "telegram", + recipients: ["-1001", "-1002"], + }); + await waitFor("telegram", () => telegram.for(a.id).length === 2, 10_000); + expect( + telegram + .for(a.id) + .map((t) => t.destination) + .sort(), + ).toEqual(["-1001", "-1002"]); + + await a.call("send_campaign", { + campaign: "room", + template: "note", + channel: "slack", + recipients: ["C123"], + }); + await waitFor("slack", () => slack.for(a.id).length === 1, 10_000); + + // A user given the same address twice, once as `email` and once as a contact. + const both = await a.call("upsert_user", { + id: "twice", + email: "t@x.com", + contacts: [{ channel: "email", target: "t@x.com" }], + }); + expect(both.isError).toBe(false); + + expect((await a.call("send_campaign", { campaign: "x", template: "promo" })).isError).toBe( + true, + ); + await a.close(); + }); + + it("previews a template exactly as the pipeline renders it", async () => { + const a = await projectAgent(); + const content = { + subject: "Re: {{topic}}", + html: "

{{body}}

", + text: "{{body}} / {{{body}}}", + }; + await a.call("upsert_template", { id: "rich", channel: "email", content }); + await a.call("upsert_user", { id: "u", email: "u@x.com" }); + const data = { topic: "a\r\nBcc: evil", body: "bold" }; + + const preview = await a.call("preview_template", { id: "rich", data: { ...data, extra: 1 } }); + await a.call("send_notification", { template: "rich", user: "u", data }); + await waitFor("sent", () => email.for(a.id).length === 1); + + expect(preview.body.rendered).toEqual(email.for(a.id)[0]!.renderedContent.content); + expect(preview.body).toMatchObject({ templateId: "rich", channel: "email" }); + expect(preview.body.resolvedVariables.sort()).toEqual(["body", "topic"]); + + const raw = await a.call("render_template", { content: { text: "{{missing}}!" } }); + expect(raw.body).toMatchObject({ rendered: { text: "!" }, unresolvedVariables: ["missing"] }); + expect((await a.call("preview_template", {})).isError).toBe(true); + await a.close(); + }); + + it("manages templates", async () => { + const a = await projectAgent(); + await a.call("upsert_template", { + id: "t1", + channel: "email", + topic: "news", + content: { subject: "s" }, + }); + await a.call("upsert_template", { id: "t2", channel: "sms", content: { text: "x" } }); + expect( + (await a.call("list_templates", { channel: "sms" })).body.templates.map((t: any) => t.id), + ).toEqual(["t2"]); + expect( + (await a.call("list_templates", { topic: "news" })).body.templates.map((t: any) => t.id), + ).toEqual(["t1"]); + expect((await a.call("get_template", { id: "t1" })).body.topics).toEqual(["news"]); + expect((await a.call("delete_template", { id: "t1" })).isError).toBe(false); + const gone = await a.call("get_template", { id: "t1" }); + expect(gone.isError).toBe(true); + expect(gone.text).toContain("HTTP 404"); + await a.close(); + }); + + it("manages users, contacts and preferences", async () => { + const a = await projectAgent(); + await a.call("upsert_user", { + id: "u", + email: "u@x.com", + language: "de", + segments: ["vip"], + preferences: { topics: { news: true } }, + }); + await a.call("update_user", { id: "u", timezone: "Asia/Tokyo" }); + await a.call("add_user_contact", { userId: "u", channel: "sms", target: "+1555" }); + + const user = (await a.call("get_user", { id: "u" })).body; + expect(user).toMatchObject({ + userId: "u", + language: "de", + timezone: "Asia/Tokyo", + segments: ["vip"], + }); + expect( + (await a.call("get_user_contacts", { userId: "u" })).body.contacts + .map((c: any) => c.target) + .sort(), + ).toEqual(["+1555", "u@x.com"]); + expect( + (await a.call("list_users", { segment: "vip" })).body.users.map((u: any) => u.userId), + ).toEqual(["u"]); + expect((await a.call("list_segments")).body.segments).toEqual(["vip"]); + + await a.call("update_user_preferences", { + userId: "u", + topics: { news: false }, + quietHours: [{ start: "22:00", end: "06:00" }], + }); + expect((await a.call("get_user_preferences", { userId: "u" })).body).toMatchObject({ + topics: { news: false }, + quietHours: [{ start: "22:00", end: "06:00" }], + }); + + await a.call("delete_user_contact", { userId: "u", channel: "sms", target: "+1555" }); + expect((await a.call("get_user_contacts", { userId: "u" })).body.contacts).toHaveLength(1); + await a.call("delete_user", { id: "u" }); + expect((await a.call("get_user", { id: "u" })).isError).toBe(true); + await a.close(); + }); + + it("manages suppressions", async () => { + const a = await projectAgent(); + await a.call("suppress_address", { + channel: "email", + target: "Stop@X.com", + reason: "complained", + }); + expect( + (await a.call("list_suppressions", { channel: "email" })).body.suppressions.map( + (s: any) => s.target, + ), + ).toEqual(["stop@x.com"]); + await a.call("unsuppress_address", { channel: "email", target: "stop@x.com" }); + expect((await a.call("list_suppressions", {})).body.suppressions).toEqual([]); + await a.close(); + }); + + it("lists, inspects and cancels scheduled sends, and reads delivery logs", async () => { + const a = await projectAgent(); + await a.call("upsert_template", { id: "t", channel: "email", content: { subject: "s" } }); + await a.call("upsert_user", { id: "u", email: "u@x.com" }); + await a.call("send_notification", { + template: "t", + user: "u", + sendAt: new Date(Date.now() + 3600_000).toISOString(), + }); + await a.call("send_notification", { template: "t", user: "u" }); + + let scheduled: any[] = []; + await waitFor("parked", async () => { + scheduled = (await a.call("list_scheduled", {})).body.scheduled ?? []; + return scheduled.length === 1; + }); + expect((await a.call("cancel_notification", { taskId: scheduled[0].taskId })).isError).toBe( + false, + ); + expect((await a.call("list_scheduled", {})).body.scheduled).toEqual([]); + + await waitFor( + "logged", + async () => + (await a.call("get_delivery_logs", { status: "delivered" })).body.logs?.length === 1, + ); + const log = (await a.call("get_delivery_logs", { status: "delivered" })).body.logs[0]; + expect((await a.call("get_notification", { taskId: log.taskId })).body.status).toBe( + "delivered", + ); + await a.close(); + }); + + it("defines, runs, inspects and cancels workflows, and ingests events", async () => { + const a = await projectAgent(); + await a.call("upsert_template", { id: "t", channel: "email", content: { subject: "s" } }); + await a.call("upsert_user", { id: "u", email: "u@x.com" }); + await a.call("create_workflow", { + name: "drip", + steps: [ + { action: "notify", payload: { template: "t" } }, + { action: "waitForEvent", event: "clicked", options: { timeout: "1h" } }, + ], + }); + expect( + (await a.call("list_workflows", { search: "dri" })).body.workflows.map((w: any) => w.name), + ).toEqual(["drip"]); + + const run = (await a.call("trigger_workflow", { name: "drip", user: "u" })).body; + await waitFor( + "waiting", + async () => + (await a.call("get_workflow_run", { instanceId: run.instanceId })).body.status === + "pending", + ); + await waitFor("first step sent", () => email.for(a.id).length === 1); + + expect((await a.call("ingest_event", { name: "unrelated", properties: {} })).isError).toBe( + false, + ); + expect((await a.call("cancel_workflow_run", { instanceId: run.instanceId })).isError).toBe( + false, + ); + expect((await a.call("get_workflow_run", { instanceId: run.instanceId })).body.status).toBe( + "canceled", + ); + await a.close(); + }); + + it("reports system health, metrics and dead letters", async () => { + const a = await projectAgent(); + expect((await a.call("get_system_health")).body.status).toBe("healthy"); + expect((await a.call("get_system_metrics")).body.deliveryStats).toBeDefined(); + + const ev = { + id: randomUUID(), + type: "notification.dispatched", + timestamp: new Date().toISOString(), + payload: { projectId: a.id }, + metadata: { traceId: "t", source: "s", retryCount: 0 }, + dlq: { + originalStream: STREAMS.OUTBOUND_NORMAL, + ackedAt: new Date().toISOString(), + reason: "x", + }, + }; + const id1 = await infra.redis.xadd(STREAMS.DEAD_LETTER, "*", "data", JSON.stringify(ev)); + const id2 = await infra.redis.xadd( + STREAMS.DEAD_LETTER, + "*", + "data", + JSON.stringify({ ...ev, id: randomUUID() }), + ); + const dead = (await a.call("get_dead_letters")).body.messages.map((m: any) => m.id); + expect(dead).toEqual(expect.arrayContaining([id1, id2])); + + expect((await a.call("delete_dead_letter", { id: id1 })).isError).toBe(false); + expect((await a.call("replay_dead_letter", { id: id2 })).body).toMatchObject({ + success: true, + stream: STREAMS.OUTBOUND_NORMAL, + }); + await a.close(); + }); + + it("administers projects and keys with the admin key", async () => { + const admin = await agent(ADMIN_KEY); + const created = (await admin.call("create_project", { name: "Via MCP" })).body; + expect(created.apiKey).toMatch(/^nk_live_/); + expect((await admin.call("list_projects")).body.projects.map((p: any) => p.id)).toContain( + created.id, + ); + await admin.call("update_project", { id: created.id, throttleLimit: 3 }); + + const key = ( + await admin.call("create_project_key", { projectId: created.id, role: "read_only" }) + ).body; + expect( + (await admin.call("list_project_keys", { projectId: created.id })).body.keys, + ).toHaveLength(2); + expect( + (await admin.call("delete_project_key", { projectId: created.id, keyId: key.id })).isError, + ).toBe(false); + expect((await admin.call("delete_project", { id: created.id })).isError).toBe(false); + expect((await admin.call("list_projects")).body.projects.map((p: any) => p.id)).not.toContain( + created.id, + ); + await admin.close(); + }); + + it("reports an API failure to the model instead of crashing", async () => { + const bad = await agent("nk_live_not_a_key"); + const res = await bad.call("list_templates", {}); + expect(res.isError).toBe(true); + expect(res.text).toContain("HTTP 401"); + await bad.close(); + }); +}); + +describe("NotifkitApi (the MCP server's HTTP client)", () => { + it("talks to a real server: query strings, bodies, 204s and errors", async () => { + const p = await app.project(); + const api = new NotifkitApi({ baseUrl: `${app.baseUrl}/`, apiKey: p.apiKey }); + await api.put("/v1/templates", { + templates: [ + { id: "a", channel: "email", content: {} }, + { id: "b", channel: "sms", content: {} }, + ], + }); + expect( + ( + await api.get("/v1/templates", { channel: "sms", limit: 5, topic: undefined }) + ).templates.map((t: any) => t.id), + ).toEqual(["b"]); + await api.post("/v1/users", { id: "u" }); + await api.patch("/v1/users/u", { language: "fr" }); + expect(await api.delete("/v1/templates/a")).toBeUndefined(); + + const err: any = await api.get("/v1/templates/a").catch((e) => e); + expect(err).toBeInstanceOf(NotifkitApiError); + expect(err.status).toBe(404); + expect(err.body).toMatchObject({ error: "template_not_found" }); + + const admin = new NotifkitApi({ baseUrl: app.baseUrl, apiKey: ADMIN_KEY, projectId: p.id }); + expect((await admin.get("/v1/users/u")).language).toBe("fr"); + }); +}); diff --git a/tests/integration/pipeline-fused.test.ts b/tests/integration/pipeline-fused.test.ts new file mode 100644 index 0000000..82ac9ea --- /dev/null +++ b/tests/integration/pipeline-fused.test.ts @@ -0,0 +1,3 @@ +import { definePipelineSuite } from "./support/pipeline-cases.js"; + +definePipelineSuite({ fused: true }); diff --git a/tests/integration/pipeline-streamed.test.ts b/tests/integration/pipeline-streamed.test.ts new file mode 100644 index 0000000..554b0a2 --- /dev/null +++ b/tests/integration/pipeline-streamed.test.ts @@ -0,0 +1,3 @@ +import { definePipelineSuite } from "./support/pipeline-cases.js"; + +definePipelineSuite({ fused: false }); diff --git a/tests/integration/providers.test.ts b/tests/integration/providers.test.ts new file mode 100644 index 0000000..f5eb8c0 --- /dev/null +++ b/tests/integration/providers.test.ts @@ -0,0 +1,646 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; +import { createHmac, randomBytes, randomUUID } from "node:crypto"; +import { TelegramTransport } from "../../packages/provider-telegram/src/index.js"; +import { DiscordTransport } from "../../packages/provider-discord/src/index.js"; +import { SlackTransport } from "../../packages/provider-slack/src/index.js"; +import { WhatsAppTransport } from "../../packages/provider-whatsapp/src/index.js"; +import { ResendTransport } from "../../packages/provider-resend/src/index.js"; +import { TwilioTransport } from "../../packages/provider-twilio/src/index.js"; +import { ConsoleTransport } from "../../packages/provider-console/src/index.js"; +import type { NotificationDispatchedPayload } from "@/contracts/index.js"; +import { startFakeInternet, type FakeInternet } from "./support/fake-internet.js"; +import { useInfra, waitFor } from "./support/infra.js"; +import { startNotifkit, apiClient, type Running } from "./support/pipeline.js"; + +/** + * Each provider sends through its real code — real fetch, real request + * bodies, real status handling — against a local stand-in for the vendor's + * API. Webhook signatures are computed the way each vendor documents them, + * not by the code under test. + */ + +let net: FakeInternet; +beforeAll(async () => { + net = await startFakeInternet(); +}); +afterAll(async () => { + await net?.close(); +}); +beforeEach(() => { + net.requests.length = 0; +}); + +function task( + overrides: Partial & { content?: Record } = {}, +): NotificationDispatchedPayload { + const { content, ...rest } = overrides; + return { + projectId: randomUUID(), + taskId: `task-${randomUUID()}`, + enrichedEventId: randomUUID(), + recipientId: "u1", + channel: "email", + priority: "normal", + templateVariables: {}, + deliveryOptions: { maxAttempts: 3, timeoutMs: 2_000 }, + destination: "dest", + renderedContent: { content: content ?? { subject: "Hello", text: "Body" } }, + ...rest, + } as NotificationDispatchedPayload; +} + +// ─── Telegram ─────────────────────────────────────────────────────────────── + +describe("TelegramTransport", () => { + const t = new TelegramTransport({ botToken: "123:abc", parseMode: "HTML" }); + + it("posts the message to the bot API and returns its message id", async () => { + net.on("api.telegram.org", () => ({ json: { ok: true, result: { message_id: 77 } } })); + const r = await t.send( + task({ + destination: "-100200", + content: { subject: "Alert", text: "Disk full", disableNotification: true }, + }), + ); + + expect(r).toEqual({ success: true, providerMessageId: "77" }); + const [req] = net.requests; + expect(req!.method).toBe("POST"); + expect(req!.path).toBe("/bot123:abc/sendMessage"); + expect(req!.json()).toEqual({ + chat_id: "-100200", + text: "Alert\n\nDisk full", + parse_mode: "HTML", + disable_notification: true, + }); + }); + + it("sends a subject-only template as just the subject", async () => { + net.on("api.telegram.org", () => ({ json: { ok: true, result: { message_id: 1 } } })); + await t.send(task({ content: { subject: "Only a subject" } })); + expect(net.requests[0]!.json().text).toBe("Only a subject"); + }); + + it("marks a blocked bot or vanished chat as a dead destination", async () => { + net.on("api.telegram.org", () => ({ + status: 403, + json: { ok: false, error_code: 403, description: "Forbidden: bot was blocked by the user" }, + })); + expect(await t.send(task())).toMatchObject({ + success: false, + invalidToken: true, + error: "Forbidden: bot was blocked by the user", + }); + + net.on("api.telegram.org", () => ({ + status: 400, + json: { ok: false, error_code: 400, description: "Bad Request: chat not found" }, + })); + expect((await t.send(task())).invalidToken).toBe(true); + }); + + it("treats a rate limit as retryable, not as a dead destination", async () => { + net.on("api.telegram.org", () => ({ + status: 429, + json: { ok: false, error_code: 429, description: "Too Many Requests: retry after 5" }, + })); + expect(await t.send(task())).toMatchObject({ success: false, invalidToken: false }); + }); + + it("reports a network failure or a non-JSON gateway page as a failed send", async () => { + net.on("api.telegram.org", () => "hang-up"); + expect((await t.send(task())).success).toBe(false); + net.on("api.telegram.org", () => ({ status: 502, text: "Bad Gateway" })); + expect((await t.send(task())).success).toBe(false); + }); + + it("refuses a task with no chat id without calling the API", async () => { + expect((await t.send(task({ destination: undefined }))).success).toBe(false); + expect(net.requests).toEqual([]); + }); +}); + +// ─── Discord ──────────────────────────────────────────────────────────────── + +describe("DiscordTransport", () => { + const d = new DiscordTransport({ username: "Alerts" }); + const hook = "https://discord.com/api/webhooks/1/abc"; + + it("posts to the webhook, waiting for the created message's id", async () => { + net.on("discord.com", () => ({ json: { id: "msg-9" } })); + const r = await d.send( + task({ + destination: hook, + content: { subject: "Deploy", text: "done", avatarUrl: "https://x/a.png" }, + }), + ); + expect(r).toEqual({ success: true, providerMessageId: "msg-9" }); + expect(net.requests[0]!.path).toBe("/api/webhooks/1/abc?wait=true"); + expect(net.requests[0]!.json()).toEqual({ + content: "**Deploy**\ndone", + username: "Alerts", + avatar_url: "https://x/a.png", + }); + }); + + it("sends a subject-only template as just the subject", async () => { + net.on("discord.com", () => ({ json: { id: "m" } })); + await d.send(task({ destination: hook, content: { subject: "Heads up" } })); + expect(net.requests[0]!.json().content).toBe("**Heads up**"); + }); + + it("refuses to post anywhere but Discord", async () => { + for (const bad of [ + "http://discord.com/api/webhooks/1/a", + "https://evil.com/api/webhooks/1/a", + "https://discord.com/other/1", + ]) { + expect(await d.send(task({ destination: bad }))).toMatchObject({ + success: false, + invalidToken: true, + }); + } + expect(net.requests).toEqual([]); + }); + + it("marks a deleted webhook dead and a server error retryable", async () => { + net.on("discord.com", () => ({ + status: 404, + json: { message: "Unknown Webhook", code: 10015 }, + })); + expect(await d.send(task({ destination: hook }))).toMatchObject({ + success: false, + invalidToken: true, + }); + net.on("discord.com", () => ({ status: 500, text: "oops" })); + expect(await d.send(task({ destination: hook }))).toMatchObject({ + success: false, + invalidToken: false, + }); + }); +}); + +// ─── Slack ────────────────────────────────────────────────────────────────── + +describe("SlackTransport", () => { + it("posts to chat.postMessage with the bot token", async () => { + net.on("slack.com", () => ({ json: { ok: true, channel: "C1", ts: "1700.1" } })); + const s = new SlackTransport({ botToken: "xoxb-1" }); + const r = await s.send( + task({ destination: "C1", content: { text: "hi", blocks: [{ type: "divider" }] } }), + ); + expect(r).toEqual({ success: true, providerMessageId: "C1:1700.1" }); + const req = net.requests[0]!; + expect(req.path).toBe("/api/chat.postMessage"); + expect(req.headers.authorization).toBe("Bearer xoxb-1"); + expect(req.json()).toEqual({ channel: "C1", text: "hi", blocks: [{ type: "divider" }] }); + }); + + it("uses a template's own channel over the recipient's", async () => { + net.on("slack.com", () => ({ json: { ok: true, ts: "1" } })); + await new SlackTransport({ token: "xoxb" }).send( + task({ destination: "D-user", content: { channel: "C-alerts", subject: "S" } }), + ); + expect(net.requests[0]!.json()).toMatchObject({ channel: "C-alerts", text: "S" }); + }); + + it("marks an archived channel dead, but not a rate limit", async () => { + const s = new SlackTransport({ botToken: "xoxb" }); + net.on("slack.com", () => ({ json: { ok: false, error: "is_archived" } })); + expect(await s.send(task({ destination: "C" }))).toMatchObject({ + success: false, + invalidToken: true, + }); + net.on("slack.com", () => ({ json: { ok: false, error: "ratelimited" } })); + expect(await s.send(task({ destination: "C" }))).toMatchObject({ + success: false, + invalidToken: false, + }); + }); + + it("posts to an incoming webhook, and reads its plain-text errors", async () => { + const s = new SlackTransport({ webhookUrl: "https://hooks.slack.com/services/T/B/x" }); + net.on("hooks.slack.com", () => ({ text: "ok" })); + expect( + await s.send(task({ destination: undefined, content: { text: "hello" } })), + ).toMatchObject({ success: true }); + expect(net.requests[0]!.json()).toEqual({ text: "hello" }); + + net.on("hooks.slack.com", () => ({ status: 404, text: "channel_not_found" })); + expect(await s.send(task({ destination: undefined }))).toMatchObject({ + success: false, + invalidToken: true, + }); + net.on("hooks.slack.com", () => "hang-up"); + expect((await s.send(task({ destination: undefined }))).success).toBe(false); + }); + + it("explains, rather than calls Slack, when it has no way to send", async () => { + expect((await new SlackTransport().send(task({ destination: undefined }))).error).toMatch( + /No destination/, + ); + expect((await new SlackTransport().send(task({ destination: "C1" }))).error).toMatch( + /no botToken/, + ); + expect(net.requests).toEqual([]); + }); +}); + +// ─── WhatsApp ─────────────────────────────────────────────────────────────── + +describe("WhatsAppTransport", () => { + const w = new WhatsAppTransport({ + phoneNumberId: "PN1", + accessToken: "tok", + verifyToken: "vt", + appSecret: "shh", + }); + + it("sends a text message to the digits of the number", async () => { + net.on("graph.facebook.com", () => ({ json: { messages: [{ id: "wamid.1" }] } })); + const r = await w.send( + task({ destination: "+1 (555) 010-0200", content: { text: "Your code is 1" } }), + ); + expect(r).toEqual({ success: true, providerMessageId: "wamid.1" }); + const req = net.requests[0]!; + expect(req.path).toBe("/v21.0/PN1/messages"); + expect(req.headers.authorization).toBe("Bearer tok"); + expect(req.json()).toEqual({ + messaging_product: "whatsapp", + to: "15550100200", + type: "text", + text: { body: "Your code is 1" }, + }); + }); + + it("flags an expired access token, and gives up on a hung API within the task's timeout", async () => { + net.on("graph.facebook.com", () => ({ + status: 401, + json: { error: { code: 190, message: "Session expired" } }, + })); + expect(await w.send(task({ destination: "+1555", content: { text: "x" } }))).toMatchObject({ + success: false, + invalidToken: true, + error: "Session expired", + }); + + net.on("graph.facebook.com", () => ({ delayMs: 3_000, json: {} })); + const started = Date.now(); + const r = await w.send( + task({ + destination: "+1555", + content: { text: "x" }, + deliveryOptions: { maxAttempts: 3, timeoutMs: 300 }, + }), + ); + expect(r.success).toBe(false); + expect(Date.now() - started).toBeLessThan(2_000); + }); + + it("answers Meta's verification handshake only with the right token", () => { + expect( + w.verifyWebhookChallenge( + new URLSearchParams("hub.mode=subscribe&hub.verify_token=vt&hub.challenge=42"), + ), + ).toBe("42"); + expect( + w.verifyWebhookChallenge( + new URLSearchParams("hub.mode=subscribe&hub.verify_token=no&hub.challenge=42"), + ), + ).toBeUndefined(); + }); + + it("accepts only webhooks signed with the app secret", () => { + const raw = JSON.stringify({ entry: [] }); + const sig = `sha256=${createHmac("sha256", "shh").update(raw).digest("hex")}`; + expect(w.verifyWebhook(raw, { "x-hub-signature-256": sig })).toBe(true); + expect(w.verifyWebhook(raw + " ", { "x-hub-signature-256": sig })).toBe(false); + expect(w.verifyWebhook(raw, {})).toBe(false); + expect( + new WhatsAppTransport({ phoneNumberId: "p", accessToken: "t" }).verifyWebhook(raw, { + "x-hub-signature-256": sig, + }), + ).toBe(false); + }); + + it("maps read and failed statuses, ignoring the rest", async () => { + const events = await w.parseWebhook({ + entry: [ + { + changes: [ + { + value: { + statuses: [ + { id: "a", status: "read", recipient_id: "1555", timestamp: "1700000000" }, + { id: "b", status: "failed", recipient_id: "1666", errors: [{ code: 131026 }] }, + { id: "c", status: "delivered" }, + ], + }, + }, + ], + }, + ], + }); + expect(events.map((e) => [e.providerMessageId, e.status, e.recipient, e.bounceType])).toEqual([ + ["a", "opened", "1555", undefined], + ["b", "bounced", "1666", undefined], + ]); + }); +}); + +// ─── Resend ───────────────────────────────────────────────────────────────── + +/** Signs a payload the way Resend (via Svix) does. */ +function svixSign(secret: string, id: string, timestamp: number, payload: string) { + const key = Buffer.from(secret.replace(/^whsec_/, ""), "base64"); + const sig = createHmac("sha256", key).update(`${id}.${timestamp}.${payload}`).digest("base64"); + return { "svix-id": id, "svix-timestamp": String(timestamp), "svix-signature": `v1,${sig}` }; +} + +describe("ResendTransport", () => { + const secret = `whsec_${randomBytes(24).toString("base64")}`; + const r = new ResendTransport({ + apiKey: "re_test", + from: "Acme ", + webhookSecret: secret, + }); + + it("sends the rendered email with its headers and returns Resend's id", async () => { + net.on("api.resend.com", () => ({ json: { id: "re-1" } })); + const res = await r.send( + task({ + destination: "u@x.com", + content: { + subject: "Hi", + html: "Hi", + text: "Hi", + from: "Sales ", + replyTo: "help@acme.test", + }, + deliveryOptions: { + maxAttempts: 3, + timeoutMs: 1000, + headers: { "List-Unsubscribe": "" }, + }, + }), + ); + expect(res).toEqual({ success: true, providerMessageId: "re-1" }); + const req = net.requests[0]!; + expect(req.path).toBe("/emails"); + expect(req.headers.authorization).toBe("Bearer re_test"); + expect(req.json()).toMatchObject({ + from: "Sales ", + to: "u@x.com", + subject: "Hi", + html: "Hi", + text: "Hi", + reply_to: "help@acme.test", + headers: { "List-Unsubscribe": "" }, + }); + }); + + it("escapes a text-only body when it builds the HTML part", async () => { + net.on("api.resend.com", () => ({ json: { id: "re-2" } })); + await r.send( + task({ + destination: "u@x.com", + content: { subject: "S", text: 'Hi ' }, + }), + ); + const html = net.requests[0]!.json().html as string; + expect(html).not.toContain(" { + net.on("api.resend.com", () => ({ + status: 422, + json: { statusCode: 422, name: "validation_error", message: "Invalid `to` field" }, + })); + expect(await r.send(task({ destination: "nope" }))).toMatchObject({ + success: false, + error: "Invalid `to` field", + }); + }); + + it("verifies Svix-signed webhooks and maps bounces, complaints, opens and clicks", async () => { + const now = Math.floor(Date.now() / 1000); + const payload = JSON.stringify({ + type: "email.bounced", + created_at: new Date().toISOString(), + data: { + email_id: "re-1", + to: ["u@x.com"], + bounce: { type: "Permanent", message: "no such user" }, + }, + }); + const headers = svixSign(secret, "msg_1", now, payload); + + expect(r.verifyWebhook(payload, headers)).toBe(true); + const [bounce] = await r.parseWebhook(JSON.parse(payload), payload, headers); + expect(bounce).toMatchObject({ + providerMessageId: "re-1", + status: "bounced", + bounceType: "hard", + recipient: "u@x.com", + }); + + const click = JSON.stringify({ + type: "email.clicked", + data: { email_id: "re-2", to: ["u@x.com"], click: { link: "https://acme.test" } }, + }); + const [c] = await r.parseWebhook(null, click, svixSign(secret, "msg_2", now, click)); + expect(c).toMatchObject({ status: "clicked", metadata: { url: "https://acme.test" } }); + + const delivered = JSON.stringify({ type: "email.delivered", data: { email_id: "re-3" } }); + expect( + await r.parseWebhook(null, delivered, svixSign(secret, "msg_3", now, delivered)), + ).toEqual([]); + }); + + it("rejects a forged, replayed-from-long-ago or unsigned webhook", async () => { + const payload = JSON.stringify({ + type: "email.complained", + data: { email_id: "re-1", to: ["u@x.com"] }, + }); + const forged = svixSign( + `whsec_${randomBytes(24).toString("base64")}`, + "m", + Math.floor(Date.now() / 1000), + payload, + ); + expect(r.verifyWebhook(payload, forged)).toBe(false); + expect( + r.verifyWebhook( + payload, + svixSign(secret, "m", Math.floor(Date.now() / 1000) - 3600, payload), + ), + ).toBe(false); + expect(r.verifyWebhook(payload, {})).toBe(false); + expect(await r.parseWebhook(JSON.parse(payload), payload, forged)).toEqual([]); + expect( + new ResendTransport({ apiKey: "k", from: "f" }).verifyWebhook( + payload, + svixSign(secret, "m", Math.floor(Date.now() / 1000), payload), + ), + ).toBe(false); + }); +}); + +// ─── Twilio (webhook side; sending goes through Twilio's own HTTPS client) ── + +/** Twilio's documented request signature: HMAC-SHA1 over URL + sorted params. */ +function twilioSign(authToken: string, url: string, params: Record) { + const data = + url + + Object.keys(params) + .sort() + .map((k) => k + params[k]) + .join(""); + return createHmac("sha1", authToken).update(data).digest("base64"); +} + +describe("TwilioTransport webhooks", () => { + const url = "https://api.acme.test/webhooks/twilio"; + const t = new TwilioTransport({ + accountSid: "AC1", + authToken: "secret", + from: "+1000", + statusCallbackUrl: url, + }); + + it("mounts at the callback URL's path", () => { + expect(t.webhookPath).toBe("/webhooks/twilio"); + expect( + () => + new TwilioTransport({ + accountSid: "a", + authToken: "b", + from: "c", + statusCallbackUrl: "not a url", + }), + ).toThrow(); + }); + + it("verifies Twilio's signature over every posted field", async () => { + const params = { + MessageSid: "SM1", + MessageStatus: "undelivered", + To: "+1555", + ErrorCode: "30005", + NewField: "x", + }; + const raw = new URLSearchParams(params).toString(); + const sig = twilioSign("secret", url, params); + + expect(await t.verifyWebhook(raw, { "x-twilio-signature": sig })).toBe(true); + expect( + await t.verifyWebhook(raw.replace("30005", "30006"), { "x-twilio-signature": sig }), + ).toBe(false); + expect(await t.verifyWebhook(raw, {})).toBe(false); + + const [e] = await t.parseWebhook({}, raw, { "x-twilio-signature": sig }); + expect(e).toMatchObject({ + providerMessageId: "SM1", + status: "bounced", + bounceType: "hard", + recipient: "+1555", + }); + }); + + it("treats a STOP reply as an unsubscribe and ignores progress reports", async () => { + const stop = { MessageSid: "SM2", MessageStatus: "failed", To: "+1555", ErrorCode: "21610" }; + const raw = new URLSearchParams(stop).toString(); + const [e] = await t.parseWebhook({}, raw, { + "x-twilio-signature": twilioSign("secret", url, stop), + }); + expect(e!.status).toBe("unsubscribed"); + + const sent = { MessageSid: "SM3", MessageStatus: "sent" }; + const raw2 = new URLSearchParams(sent).toString(); + expect( + await t.parseWebhook({}, raw2, { "x-twilio-signature": twilioSign("secret", url, sent) }), + ).toEqual([]); + }); + + it("refuses to send without a destination or a body", async () => { + expect((await t.send(task({ destination: undefined }))).success).toBe(false); + expect((await t.send(task({ destination: "+1", content: { subject: "only" } }))).success).toBe( + false, + ); + }); +}); + +describe("ConsoleTransport", () => { + it("delivers after the configured latency and returns a message id", async () => { + const c = new ConsoleTransport({ channel: "sms", latencyMs: "50-60" }); + const started = Date.now(); + const r = await c.send(task()); + expect(Date.now() - started).toBeGreaterThanOrEqual(45); + expect(r.success).toBe(true); + expect(r.providerMessageId).toMatch(/^console-/); + expect(new ConsoleTransport().channel).toBe("push"); + }); +}); + +// ─── A provider end to end ───────────────────────────────────────────────── + +describe("Resend inside notifkit", () => { + const infra = useInfra(); + let app: Running; + const secret = `whsec_${randomBytes(24).toString("base64")}`; + + beforeAll(async () => { + app = await startNotifkit(infra, { + services: ["api", "enricher", "engine", "delivery", "events"], + providers: [ + new ResendTransport({ + apiKey: "re_live", + from: "no-reply@acme.test", + webhookSecret: secret, + }), + ], + env: { RATE_LIMIT_PER_HOUR: "100000" }, + }); + }, 120_000); + afterAll(async () => { + await app?.stop(); + }, 60_000); + + it("delivers through Resend's API, then suppresses the address when Resend reports a hard bounce", async () => { + let n = 0; + net.on("api.resend.com", () => ({ json: { id: `re-${++n}` } })); + const p = await app.project(); + await p.api("PUT", "/v1/templates", { + templates: [ + { id: "t", channel: "email", content: { subject: "Hi {{name}}", text: "Hello" } }, + ], + }); + await p.api("POST", "/v1/users", { id: "u", email: "gone@x.com" }); + await p.api("POST", "/v1/notify", { user: "u", template: "t", data: { name: "Al" } }); + + await waitFor("sent to Resend", () => net.requests.some((r) => r.host === "api.resend.com")); + const sent = net.requests.find((r) => r.host === "api.resend.com")!.json(); + expect(sent).toMatchObject({ to: "gone@x.com", subject: "Hi Al", from: "no-reply@acme.test" }); + await waitFor("logged", async () => { + const rows = + await infra.sql`SELECT 1 FROM message_logs WHERE provider_message_id = 're-1' AND status = 'delivered'`; + return rows.length === 1; + }); + + const payload = JSON.stringify({ + type: "email.bounced", + data: { email_id: "re-1", to: ["gone@x.com"], bounce: { type: "Permanent" } }, + }); + const res = await apiClient(app.baseUrl)("POST", "/webhooks/resend", payload, { + "content-type": "application/json", + ...svixSign(secret, "msg_e2e", Math.floor(Date.now() / 1000), payload), + }); + expect(res.status).toBe(200); + + const suppressions = await p.api("GET", "/v1/suppressions"); + expect(suppressions.body.suppressions.map((s: any) => [s.target, s.reason])).toEqual([ + ["gone@x.com", "bounced"], + ]); + }); +}); diff --git a/tests/integration/queue.test.ts b/tests/integration/queue.test.ts new file mode 100644 index 0000000..73c0368 --- /dev/null +++ b/tests/integration/queue.test.ts @@ -0,0 +1,401 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { randomUUID } from "node:crypto"; +import { StreamProducer, StreamConsumer, PendingMessageScanner } from "@/queue/index.js"; +import { STREAMS, CONSUMER_GROUPS, buildStreamEvent, type StreamName } from "@/contracts/index.js"; +import { createLogger } from "@/logger/index.js"; +import { useInfra, waitFor, settle } from "./support/infra.js"; + +const infra = useInfra(); +const logger = createLogger({ name: "test", level: "silent" }); + +const A = STREAMS.INBOUND_NORMAL; +const B = STREAMS.INBOUND_LOW; +const DLQ = STREAMS.DEAD_LETTER; +const GROUP = CONSUMER_GROUPS.ENRICHER; + +function event(n = 0) { + return buildStreamEvent("test.event", { n }, "test"); +} + +/** A full envelope, as it would sit in a stream. */ +function envelope(n = 0) { + return { ...event(n), id: randomUUID(), timestamp: new Date().toISOString() }; +} + +const consumers: StreamConsumer[] = []; +function consumer(opts: Partial[0]> = {}) { + const c = new StreamConsumer({ + redis: infra.redis, + stream: A, + group: GROUP, + consumer: `c-${randomUUID().slice(0, 6)}`, + blockMs: 100, + logger, + ...opts, + }); + consumers.push(c); + return c; +} + +/** Reads until `count` messages arrived or the timeout passes. */ +async function readN(c: StreamConsumer, count: number, timeoutMs = 5_000) { + const got: any[] = []; + const it = c.readBatch(); + const deadline = Date.now() + timeoutMs; + while (got.length < count && Date.now() < deadline) { + const next = await Promise.race([ + it.next(), + new Promise<{ done: true; value: undefined }>((r) => + setTimeout(() => r({ done: true, value: undefined }), deadline - Date.now()), + ), + ]); + if (next.done) break; + got.push(...next.value); + } + await c.stop(); + return got; +} + +async function pendingCount(stream: string, group = GROUP): Promise { + const summary = (await infra.redis.xpending(stream, group)) as any[]; + return Number(summary[0]); +} + +beforeEach(async () => { + await infra.redis.flushdb(); +}); + +afterEach(async () => { + await Promise.all(consumers.splice(0).map((c) => c.stop().catch(() => {}))); +}); + +describe("StreamProducer", () => { + it("writes an envelope with a fresh id and timestamp, and returns the stream entry id", async () => { + const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 1000 }); + const entryId = await producer.publish(event(7)); + + const entries = await infra.redis.xrange(A, "-", "+"); + expect(entries).toHaveLength(1); + expect(entries[0]![0]).toBe(entryId); + const stored = JSON.parse(entries[0]![1][1]!); + expect(stored.payload).toEqual({ n: 7 }); + expect(stored.id).toMatch(/^[0-9a-f-]{36}$/); + expect(Number.isNaN(Date.parse(stored.timestamp))).toBe(false); + }); + + it("keeps every message and their order when hundreds are published at once", async () => { + const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 10_000 }); + const ids = await Promise.all( + Array.from({ length: 350 }, (_, n) => producer.publish(event(n))), + ); + + expect(new Set(ids).size).toBe(350); + const entries = await infra.redis.xrange(A, "-", "+"); + expect(entries.map((e) => JSON.parse(e[1][1]!).payload.n)).toEqual( + Array.from({ length: 350 }, (_, n) => n), + ); + }); + + it("trims the stream towards maxLen", async () => { + const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 10 }); + for (let batch = 0; batch < 10; batch++) { + await Promise.all(Array.from({ length: 100 }, (_, n) => producer.publish(event(n)))); + } + // "~" trims whole radix-tree nodes (100 entries by default), so the length + // lands near, not at, the cap — but nowhere near the 1000 written. + expect(await infra.redis.xlen(A)).toBeLessThanOrEqual(200); + }); + + it("rejects the publish when Redis refuses the write", async () => { + await infra.redis.set(A, "not a stream"); + const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 100 }); + await expect(producer.publish(event())).rejects.toThrow(/WRONGTYPE/); + }); + + it("publishBatch returns the ids of the events it actually published", async () => { + const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 1000 }); + const { messageIds, eventIds } = await producer.publishBatch([event(1), event(2), event(3)]); + + const entries = await infra.redis.xrange(A, "-", "+"); + expect(entries.map((e) => e[0])).toEqual(messageIds); + // Callers record eventIds as the notification id (workflow step.notify + // does), so each must be the id of the event that entry holds. + expect(entries.map((e) => JSON.parse(e[1][1]!).id)).toEqual(eventIds); + }); + + it("publishBatch of nothing writes nothing", async () => { + const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 1000 }); + expect(await producer.publishBatch([])).toEqual({ messageIds: [], eventIds: [] }); + expect(await infra.redis.exists(A)).toBe(0); + }); +}); + +describe("StreamConsumer", () => { + it("delivers messages published before the group existed", async () => { + await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(1))); + const c = consumer(); + await c.ensureGroup(); + + const got = await readN(c, 1); + expect(got.map((m) => m.event.payload.n)).toEqual([1]); + }); + + it("ensureGroup is safe to call repeatedly and creates missing streams", async () => { + const c = consumer({ stream: [A, B] }); + await c.ensureGroup(); + await c.ensureGroup(); + const groups = (await infra.redis.xinfo("GROUPS", B)) as any[]; + expect(groups).toHaveLength(1); + }); + + it("reads from several streams and tags each message with its stream", async () => { + const c = consumer({ stream: [A, B] }); + await c.ensureGroup(); + await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(1))); + await infra.redis.xadd(B, "*", "data", JSON.stringify(envelope(2))); + + const got = await readN(c, 2); + const byN = Object.fromEntries(got.map((m) => [m.event.payload.n, m])); + expect(byN[1]!.stream).toBe(A); + expect(byN[2].stream).toBe(B); + expect(byN[1].deliveryCount).toBe(1); + }); + + it("acks and skips entries it cannot parse instead of leaving them pending", async () => { + const c = consumer(); + await c.ensureGroup(); + await infra.redis.xadd(A, "*", "other", "x"); + await infra.redis.xadd(A, "*", "data", "{not json"); + await infra.redis.xadd(A, "*", "data", JSON.stringify({ hello: "world" })); + await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(9))); + + const got = await readN(c, 1); + expect(got.map((m) => m.event.payload.n)).toEqual([9]); + // Only the good one is still pending; it has not been acked yet. + expect(await pendingCount(A)).toBe(1); + }); + + it("ack removes the message from the pending list", async () => { + const c = consumer(); + await c.ensureGroup(); + await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope())); + const [msg] = await readN(c, 1); + expect(await pendingCount(A)).toBe(1); + + await c.ack(msg.id, msg.stream); + expect(await pendingCount(A)).toBe(0); + }); + + it("buffered acks land once the timer fires, and on flushAcks", async () => { + const c = consumer({ bufferAcks: true, ackFlushMs: 20, batchSize: 100 }); + await c.ensureGroup(); + for (let n = 0; n < 3; n++) await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(n))); + const got = await readN(c, 3); + + await c.ack(got[0].id, A); + expect(await pendingCount(A)).toBe(3); // still buffered + await waitFor("timer flush", async () => (await pendingCount(A)) === 2, 2_000); + + await c.ack([got[1].id, got[2].id], A); + await c.flushAcks(); + expect(await pendingCount(A)).toBe(0); + }); + + it("buffered acks flush immediately once a batch is full", async () => { + const c = consumer({ bufferAcks: true, ackFlushMs: 60_000, batchSize: 2 }); + await c.ensureGroup(); + for (let n = 0; n < 2; n++) await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(n))); + const got = await readN(c, 2); + + await c.ack(got[0].id, A); + await c.ack(got[1].id, A); + expect(await pendingCount(A)).toBe(0); + }); + + it("stop() flushes buffered acks", async () => { + const c = consumer({ bufferAcks: true, ackFlushMs: 60_000, batchSize: 100 }); + await c.ensureGroup(); + await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope())); + const [msg] = await readN(c, 1); + await c.ack(msg.id, A); + await c.stop(); + expect(await pendingCount(A)).toBe(0); + }); + + it("nack moves the message to the dead-letter stream and acks it", async () => { + const c = consumer({ dlqStream: DLQ }); + await c.ensureGroup(); + const original = envelope(5); + await infra.redis.xadd(A, "*", "data", JSON.stringify(original)); + const [msg] = await readN(c, 1); + + await c.nack(msg.id, msg.event, msg.stream); + + expect(await pendingCount(A)).toBe(0); + const dlq = await infra.redis.xrange(DLQ, "-", "+"); + expect(dlq).toHaveLength(1); + const dead = JSON.parse(dlq[0]![1][1]!); + expect(dead.id).toBe(original.id); + expect(dead.dlq.originalStream).toBe(A); + }); + + it("nack leaves the message pending when the dead-letter write fails", async () => { + await infra.redis.set(DLQ, "wrong type"); + const c = consumer({ dlqStream: DLQ }); + await c.ensureGroup(); + await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope())); + const [msg] = await readN(c, 1); + + await expect(c.nack(msg.id, msg.event, msg.stream)).rejects.toThrow(); + // Never acked against a DLQ write that did not land. + expect(await pendingCount(A)).toBe(1); + }); + + it("nack without a dead-letter stream just acks", async () => { + const c = consumer(); + await c.ensureGroup(); + await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope())); + const [msg] = await readN(c, 1); + await c.nack(msg.id, msg.event, msg.stream); + expect(await pendingCount(A)).toBe(0); + expect(await infra.redis.exists(DLQ)).toBe(0); + }); + + it("deadLetter records an in-process event against the consumer's stream", async () => { + const c = consumer({ dlqStream: DLQ }); + const ev = envelope(3); + await c.deadLetter(ev); + const [[, fields]] = (await infra.redis.xrange(DLQ, "-", "+")) as any; + const dead = JSON.parse(fields[1]); + expect(dead.id).toBe(ev.id); + expect(dead.dlq.originalStream).toBe(A); + }); + + it("recreates its group and keeps reading after Redis loses its data", async () => { + const c = consumer(); + await c.ensureGroup(); + const it = c.readBatch(); + + await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(1))); + const first = await it.next(); + expect(first.value![0]!.event.payload.n).toBe(1); + + await infra.redis.flushdb(); + // Give the blocked read a moment to hit NOGROUP before anything is written. + await settle(300); + await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(2))); + + const second = await Promise.race([ + it.next(), + settle(5_000).then(() => ({ value: [] as any[] })), + ]); + expect(second.value!.map((m: any) => m.event.payload.n)).toEqual([2]); + }); + + it("stop() ends the read loop", async () => { + const c = consumer(); + await c.ensureGroup(); + const it = c.readBatch(); + const pending = it.next(); + await settle(50); + await c.stop(); + const result = await Promise.race([pending, settle(3_000).then(() => "hung" as const)]); + expect(result).not.toBe("hung"); + }); +}); + +describe("PendingMessageScanner", () => { + async function deliverUnacked(stream: StreamName, n: number, consumerName = "dead-worker") { + // In production the scanner's consumer has already created the group on + // every stream it reads. + for (const s of [A, B]) + await infra.redis.xgroup("CREATE", s, GROUP, "0", "MKSTREAM").catch(() => {}); + for (let i = 0; i < n; i++) { + await infra.redis.xadd(stream, "*", "data", JSON.stringify(envelope(i))); + } + await infra.redis.xreadgroup("GROUP", GROUP, consumerName, "COUNT", n, "STREAMS", stream, ">"); + } + + function scanner(name = "rescuer", stream: StreamName | StreamName[] = [A, B]) { + return new PendingMessageScanner({ + redis: infra.redis, + stream, + group: GROUP, + consumer: name, + logger, + }); + } + + it("counts and lists pending entries across its streams", async () => { + await deliverUnacked(A, 2); + await deliverUnacked(B, 3); + const s = scanner(); + + expect(await s.getPendingCount()).toBe(5); + const entries = await s.getPendingEntries(100); + expect(entries).toHaveLength(5); + expect(entries.filter((e) => e.stream === B)).toHaveLength(3); + expect(entries.every((e) => e.consumer === "dead-worker" && e.deliveryCount === 1)).toBe(true); + expect(await s.getPendingEntries(4)).toHaveLength(4); + expect(await s.getPendingEntries(100, A)).toHaveLength(2); + }); + + it("claims only entries idle past the threshold and bumps their delivery count", async () => { + await deliverUnacked(A, 2); + const s = scanner(); + + expect(await s.autoclaim(60_000)).toEqual([]); + + await settle(150); + const claimed = await s.autoclaim(100); + expect(claimed).toHaveLength(2); + expect(claimed.every((m) => m.stream === A && m.deliveryCount === 2)).toBe(true); + const entries = await s.getPendingEntries(10); + expect(entries.every((e) => e.consumer === "rescuer")).toBe(true); + }); + + it("backs off exponentially with each redelivery", async () => { + await deliverUnacked(A, 1); + const s = scanner(); + await settle(350); + expect(await s.autoclaim(300)).toHaveLength(1); // delivered once: needs > 300ms idle + + // Delivered twice now, so it needs > 600ms idle. Checked straight after the + // claim reset its idle time, so a slow machine cannot make this pass or fail. + expect(await s.autoclaim(300)).toHaveLength(0); + await settle(700); + expect(await s.autoclaim(300)).toHaveLength(1); + }); + + it("never hands one entry to two racing scanners", async () => { + await deliverUnacked(A, 20); + await settle(150); + const [x, y] = await Promise.all([ + scanner("x").autoclaim(100, 50), + scanner("y").autoclaim(100, 50), + ]); + const ids = [...x, ...y].map((m) => m.id); + expect(ids).toHaveLength(20); + expect(new Set(ids).size).toBe(20); + }); + + it("does not claim a message from one stream using an id pending on another", async () => { + await infra.redis.xgroup("CREATE", A, GROUP, "0", "MKSTREAM"); + await infra.redis.xgroup("CREATE", B, GROUP, "0", "MKSTREAM"); + // Same entry id in both streams; only B's is pending. + await infra.redis.xadd(A, "1-1", "data", JSON.stringify(envelope(100))); + await infra.redis.xadd(B, "1-1", "data", JSON.stringify(envelope(200))); + await infra.redis.xreadgroup("GROUP", GROUP, "dead", "STREAMS", B, ">"); + await settle(150); + + const claimed = await scanner().autoclaim(100); + expect(claimed.map((m) => [m.stream, m.event.payload.n])).toEqual([[B, 200]]); + expect(await pendingCount(A)).toBe(0); + }); + + it("respects the claim limit", async () => { + await deliverUnacked(A, 10); + await settle(150); + expect(await scanner().autoclaim(100, 3)).toHaveLength(3); + }); +}); diff --git a/tests/integration/repositories.test.ts b/tests/integration/repositories.test.ts new file mode 100644 index 0000000..97ef7c6 --- /dev/null +++ b/tests/integration/repositories.test.ts @@ -0,0 +1,692 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { randomUUID } from "node:crypto"; +import { + UserRepository, + PreferenceRepository, + ContactRepository, + TemplateRepository, + ProjectRepository, + WorkflowRepository, + SegmentRepository, + AdminUserRepository, +} from "@/repositories/index.js"; +import { useInfra } from "./support/infra.js"; + +const infra = useInfra(); + +let P: string; // project under test +let OTHER: string; // a second tenant, to prove isolation + +async function newProject(name = "p") { + const id = randomUUID(); + await infra.sql`INSERT INTO projects (id, name) VALUES (${id}, ${name})`; + return id; +} + +beforeEach(async () => { + await infra.reset(); + P = await newProject("main"); + OTHER = await newProject("other"); +}); + +function user(userId: string, extra: Record = {}) { + return { userId, segments: [], preferences: {}, ...extra } as any; +} + +// ─── UserRepository ───────────────────────────────────────────────────────── + +describe("UserRepository", () => { + it("upsertFull stores the profile, segments, preferences and quiet hours", async () => { + const repo = new UserRepository(infra.db); + await repo.upsertFull(P, { + userId: "u1", + email: "a@x.com", + language: "fr", + timezone: "Europe/Paris", + segments: ["vip", "beta"], + preferences: { + topics: { marketing: false, news: true }, + channels: { sms: false }, + quietHours: [{ start: "22:00", end: "07:00" }], + }, + }); + + const rec = await new UserRepository(infra.db).findRecordById(P, "u1"); + expect(rec).toMatchObject({ + userId: "u1", + email: "a@x.com", + language: "fr", + timezone: "Europe/Paris", + preferences: { + topics: { marketing: false, news: true }, + channels: { sms: false }, + quietHours: [{ start: "22:00", end: "07:00" }], + }, + }); + expect(rec!.segments.sort()).toEqual(["beta", "vip"]); + expect(await repo.findById(P, "u1")).toEqual({ + userId: "u1", + email: "a@x.com", + language: "fr", + timezone: "Europe/Paris", + }); + }); + + it("upsertFull on an existing user updates it and replaces quiet hours", async () => { + const repo = new UserRepository(infra.db); + await repo.upsertFull( + P, + user("u1", { + email: "old@x.com", + preferences: { quietHours: [{ start: "01:00", end: "02:00" }] }, + }), + ); + await repo.upsertFull( + P, + user("u1", { email: "new@x.com", preferences: { topics: { a: false }, quietHours: [] } }), + ); + + const rec = await new UserRepository(infra.db).findRecordById(P, "u1"); + expect(rec!.email).toBe("new@x.com"); + expect(rec!.preferences.topics).toEqual({ a: false }); + expect(rec!.preferences.quietHours).toBeUndefined(); + const { n } = ( + await infra.sql`SELECT count(*)::int AS n FROM users WHERE project_id = ${P}` + )[0]!; + expect(n).toBe(1); + }); + + it("keeps the same external id separate per project", async () => { + const repo = new UserRepository(infra.db); + await repo.upsertFull(P, user("same", { email: "p@x.com" })); + await repo.upsertFull(OTHER, user("same", { email: "o@x.com" })); + expect((await repo.findById(P, "same"))!.email).toBe("p@x.com"); + expect((await repo.findById(OTHER, "same"))!.email).toBe("o@x.com"); + expect(await repo.findById(P, "nobody")).toBeNull(); + expect(await repo.findRecordById(P, "nobody")).toBeNull(); + }); + + it("findRecordsByIds returns the users that exist and serves repeats from cache", async () => { + const repo = new UserRepository(infra.db); + await repo.upsertManyFull(P, [user("a"), user("b"), user("c")]); + + const first = await repo.findRecordsByIds(P, ["a", "b", "missing"]); + expect(first.map((r) => r.userId).sort()).toEqual(["a", "b"]); + + // Once cached, a row deleted behind the repository's back is still served. + await infra.sql`DELETE FROM users WHERE external_id = 'a'`; + const second = await repo.findRecordsByIds(P, ["a", "c"]); + expect(second.map((r) => r.userId).sort()).toEqual(["a", "c"]); + expect(await repo.findRecordsByIds(P, [])).toEqual([]); + }); + + it("writes through the repository invalidate its cache", async () => { + const repo = new UserRepository(infra.db); + await repo.upsertFull(P, user("u", { email: "one@x.com" })); + expect((await repo.findRecordById(P, "u"))!.email).toBe("one@x.com"); + + await repo.updatePartial(P, "u", { email: "two@x.com" }); + expect((await repo.findRecordById(P, "u"))!.email).toBe("two@x.com"); + + await repo.upsertFull(P, user("u", { email: "three@x.com" })); + expect((await repo.findRecordById(P, "u"))!.email).toBe("three@x.com"); + + await repo.delete(P, "u"); + expect(await repo.findRecordById(P, "u")).toBeNull(); + }); + + it("upsertManyFull inserts many users, de-duplicating repeated rows", async () => { + const repo = new UserRepository(infra.db); + await repo.upsertManyFull(P, [ + user("a", { segments: ["s", "s"], preferences: { topics: { t: true } } }), + user("a", { segments: ["s"], preferences: { topics: { t: false } } }), + user("b", { + preferences: { channels: { email: false }, quietHours: [{ start: "10:00", end: "11:00" }] }, + }), + ]); + + const [a, b] = await Promise.all([repo.findRecordById(P, "a"), repo.findRecordById(P, "b")]); + expect(a!.segments).toEqual(["s"]); + expect(a!.preferences.topics).toEqual({ t: false }); + expect(b!.preferences.channels).toEqual({ email: false }); + expect(b!.preferences.quietHours).toEqual([{ start: "10:00", end: "11:00" }]); + await repo.upsertManyFull(P, []); + }); + + it("upsertManyFull replaces quiet hours only for users that supplied them", async () => { + const repo = new UserRepository(infra.db); + await repo.upsertManyFull(P, [ + user("a", { preferences: { quietHours: [{ start: "01:00", end: "02:00" }] } }), + user("b", { preferences: { quietHours: [{ start: "03:00", end: "04:00" }] } }), + ]); + await repo.upsertManyFull(P, [user("a", { preferences: { quietHours: [] } }), user("b")]); + + const fresh = new UserRepository(infra.db); + expect((await fresh.findRecordById(P, "a"))!.preferences.quietHours).toBeUndefined(); + expect((await fresh.findRecordById(P, "b"))!.preferences.quietHours).toEqual([ + { start: "03:00", end: "04:00" }, + ]); + }); + + it("updatePartial changes only what it is given, and replaces segments", async () => { + const repo = new UserRepository(infra.db); + await repo.upsertFull( + P, + user("u", { email: "e@x.com", language: "en", timezone: "UTC", segments: ["a", "b"] }), + ); + + expect( + await repo.updatePartial(P, "u", { + language: "de", + segments: ["c"], + preferences: { + topics: { x: false }, + channels: { push: false }, + quietHours: [{ start: "05:00", end: "06:00" }], + }, + }), + ).toBe(true); + const rec = await new UserRepository(infra.db).findRecordById(P, "u"); + expect(rec).toMatchObject({ + email: "e@x.com", + language: "de", + timezone: "UTC", + segments: ["c"], + preferences: { + topics: { x: false }, + channels: { push: false }, + quietHours: [{ start: "05:00", end: "06:00" }], + }, + }); + + expect(await repo.updatePartial(P, "missing", { language: "x" })).toBe(false); + expect(await repo.updatePartial(OTHER, "u", { language: "x" })).toBe(false); + }); + + it("updatePartial can clear an email with null", async () => { + const repo = new UserRepository(infra.db); + await repo.upsertFull(P, user("u", { email: "e@x.com" })); + await repo.updatePartial(P, "u", { email: null }); + expect((await repo.findById(P, "u"))!.email ?? null).toBeNull(); + }); + + it("delete removes the user and everything hanging off it, in its own project only", async () => { + const repo = new UserRepository(infra.db); + const contacts = new ContactRepository(infra.db); + await repo.upsertFull(P, user("u", { segments: ["s"] })); + await repo.upsertFull(OTHER, user("u")); + await contacts.upsert(P, "u", "email", "u@x.com", { topics: { t: false } }); + + expect(await repo.delete(P, "u")).toBe(true); + expect(await repo.delete(P, "u")).toBe(false); + expect(await repo.findById(OTHER, "u")).not.toBeNull(); + const { n } = (await infra.sql`SELECT count(*)::int AS n FROM user_contacts`)[0]!; + expect(n).toBe(0); + }); + + describe("list", () => { + it("filters by search, segment, language, timezone and channel", async () => { + const repo = new UserRepository(infra.db); + await repo.upsertFull( + P, + user("alice", { + email: "alice@corp.com", + language: "en", + timezone: "UTC", + segments: ["vip"], + }), + ); + await repo.upsertFull( + P, + user("bob", { email: "bob@home.net", language: "fr", timezone: "Europe/Paris" }), + ); + await new ContactRepository(infra.db).upsert(P, "bob", "sms", "+100"); + await repo.upsertFull(OTHER, user("alice-other")); + + const ids = async (f: any) => + (await repo.list(P, 50, undefined, f)).users.map((u) => u.userId).sort(); + expect(await ids(undefined)).toEqual(["alice", "bob"]); + expect(await ids({ search: "CORP" })).toEqual(["alice"]); + expect(await ids({ search: "bo" })).toEqual(["bob"]); + expect(await ids({ segment: "vip" })).toEqual(["alice"]); + expect(await ids({ language: "fr" })).toEqual(["bob"]); + expect(await ids({ timezone: "UTC" })).toEqual(["alice"]); + expect(await ids({ channel: "sms" })).toEqual(["bob"]); + }); + + it("pages through every user, including users imported in one batch", async () => { + const repo = new UserRepository(infra.db); + // A bulk import writes every row in one transaction, so they all share + // one created_at. + await repo.upsertManyFull( + P, + Array.from({ length: 25 }, (_, i) => user(`bulk-${i}`)), + ); + + const seen: string[] = []; + let cursor: string | undefined; + for (let page = 0; page < 10; page++) { + const { users, nextCursor } = await repo.list(P, 10, cursor); + seen.push(...users.map((u) => u.userId)); + if (!nextCursor) break; + cursor = nextCursor; + } + expect(seen).toHaveLength(25); + expect(new Set(seen).size).toBe(25); + }); + }); + + it("findUsersBySegment and findUsersByTopic stay inside the project", async () => { + const repo = new UserRepository(infra.db); + await repo.upsertFull( + P, + user("a", { segments: ["vip"], preferences: { topics: { news: true } } }), + ); + await repo.upsertFull( + P, + user("b", { segments: ["vip"], preferences: { topics: { news: false } } }), + ); + await repo.upsertFull( + OTHER, + user("c", { segments: ["vip"], preferences: { topics: { news: true } } }), + ); + + expect((await repo.findUsersBySegment(P, "vip")).sort()).toEqual(["a", "b"]); + expect(await repo.findUsersBySegment(P, "none")).toEqual([]); + expect(await repo.findUsersByTopic(P, "news")).toEqual(["a"]); + }); +}); + +// ─── PreferenceRepository ─────────────────────────────────────────────────── + +describe("PreferenceRepository", () => { + it("reports topic opt-ins, defaulting to opted in", async () => { + await new UserRepository(infra.db).upsertFull( + P, + user("u", { preferences: { topics: { promo: false, news: true } } }), + ); + const prefs = new PreferenceRepository(infra.db); + + expect(await prefs.isOptedIn(P, "u", "promo")).toBe(false); + expect(await prefs.isOptedIn(P, "u", "news")).toBe(true); + expect(await prefs.isOptedIn(P, "u", "unknown")).toBe(true); + expect( + (await prefs.findByUserId(P, "u")).sort((a, b) => a.eventType.localeCompare(b.eventType)), + ).toEqual([ + { userId: "u", eventType: "news", optedIn: true }, + { userId: "u", eventType: "promo", optedIn: false }, + ]); + expect(await prefs.findByUserId(P, "nobody")).toEqual([]); + expect(await prefs.findByUserId(OTHER, "u")).toEqual([]); + }); +}); + +// ─── ContactRepository ────────────────────────────────────────────────────── + +describe("ContactRepository", () => { + beforeEach(async () => { + const users = new UserRepository(infra.db); + await users.upsertManyFull(P, [user("u"), user("v")]); + await users.upsertFull(OTHER, user("u")); + }); + + it("upsert stores a contact with its topic preferences", async () => { + const repo = new ContactRepository(infra.db); + await repo.upsert(P, "u", "email", "u@x.com", { topics: { promo: false } }); + await repo.upsert(P, "u", "email", "u@x.com", { topics: { promo: true, news: false } }); + + const contacts = await repo.findByUserId(P, "u"); + expect(contacts).toHaveLength(1); + expect(contacts[0]).toMatchObject({ + userId: "u", + channel: "email", + target: "u@x.com", + active: true, + preferences: { topics: { promo: true, news: false } }, + }); + expect(await repo.findByUserId(P, "v")).toEqual([]); + expect(await repo.findByUserId(P, "nobody")).toEqual([]); + }); + + it("upsert for an unknown user is a no-op", async () => { + const repo = new ContactRepository(infra.db); + await repo.upsert(P, "ghost", "sms", "+1"); + const { n } = (await infra.sql`SELECT count(*)::int AS n FROM user_contacts`)[0]!; + expect(n).toBe(0); + }); + + it("findActiveByUserIds returns only enabled contacts, grouped by user", async () => { + const repo = new ContactRepository(infra.db); + await repo.upsert(P, "u", "email", "u@x.com"); + await repo.upsert(P, "u", "sms", "+1"); + await repo.upsert(P, "v", "push", "tok"); + await repo.upsert(OTHER, "u", "email", "other@x.com"); + await repo.deactivate(P, "u", "sms", "+1"); + + const map = await new ContactRepository(infra.db).findActiveByUserIds(P, ["u", "v", "nobody"]); + expect(map.get("u")!.map((c) => c.target)).toEqual(["u@x.com"]); + expect(map.get("v")!.map((c) => c.channel)).toEqual(["push"]); + expect(map.has("nobody")).toBe(false); + expect((await repo.findActiveByUserIds(P, [])).size).toBe(0); + }); + + it("carries each contact's topic opt-outs, so a per-address unsubscribe can be honoured", async () => { + const repo = new ContactRepository(infra.db); + await repo.upsert(P, "u", "email", "work@x.com", { topics: { promo: false } }); + const [contact] = (await repo.findActiveByUserIds(P, ["u"])).get("u")!; + expect(contact!.preferences.topics).toEqual({ promo: false }); + }); + + it("sees writes made through the repository immediately", async () => { + const repo = new ContactRepository(infra.db); + await repo.upsert(P, "u", "email", "a@x.com"); + expect((await repo.findActiveByUserIds(P, ["u"])).get("u")).toHaveLength(1); + + await repo.upsert(P, "u", "sms", "+2"); + expect((await repo.findActiveByUserIds(P, ["u"])).get("u")).toHaveLength(2); + + await repo.deactivate(P, "u", "sms", "+2"); + expect((await repo.findActiveByUserIds(P, ["u"])).get("u")).toHaveLength(1); + + await repo.delete(P, "u", "email", "a@x.com"); + expect((await repo.findActiveByUserIds(P, ["u"])).has("u")).toBe(false); + + await repo.upsertMany(P, [{ userId: "u", channel: "email", target: "b@x.com" }]); + expect((await repo.findActiveByUserIds(P, ["u"])).get("u")).toHaveLength(1); + }); + + it("re-adding a deactivated contact turns it back on", async () => { + const repo = new ContactRepository(infra.db); + await repo.upsert(P, "u", "push", "tok"); + expect(await repo.deactivate(P, "u", "push", "tok")).toBe(true); + expect((await repo.findByUserId(P, "u"))[0]!.active).toBe(false); + await repo.upsert(P, "u", "push", "tok"); + expect((await repo.findByUserId(P, "u"))[0]!.active).toBe(true); + }); + + it("deactivate and delete report whether anything matched", async () => { + const repo = new ContactRepository(infra.db); + await repo.upsert(P, "u", "email", "u@x.com"); + expect(await repo.deactivate(P, "u", "email", "nope@x.com")).toBe(false); + expect(await repo.deactivate(P, "ghost", "email", "u@x.com")).toBe(false); + expect(await repo.delete(OTHER, "u", "email", "u@x.com")).toBe(false); + expect(await repo.delete(P, "ghost", "email", "u@x.com")).toBe(false); + expect(await repo.delete(P, "u", "email", "u@x.com")).toBe(true); + expect(await repo.findByUserId(P, "u")).toEqual([]); + }); + + it("upsertMany adds contacts for known users with their topics, skipping unknown users", async () => { + const repo = new ContactRepository(infra.db); + await repo.upsertMany(P, [ + { + userId: "u", + channel: "email", + target: "u@x.com", + preferences: { topics: { promo: false } }, + }, + { userId: "v", channel: "sms", target: "+9" }, + { userId: "ghost", channel: "sms", target: "+0" }, + ]); + await repo.upsertMany(P, []); + + expect((await repo.findByUserId(P, "u"))[0]!.preferences.topics).toEqual({ promo: false }); + expect((await repo.findByUserId(P, "v"))[0]!.target).toBe("+9"); + const { n } = (await infra.sql`SELECT count(*)::int AS n FROM user_contacts`)[0]!; + expect(n).toBe(2); + await repo.upsertMany(P, [{ userId: "ghost", channel: "sms", target: "+0" }]); + }); +}); + +// ─── TemplateRepository ───────────────────────────────────────────────────── + +describe("TemplateRepository", () => { + it("upserts, reads, lists and deletes templates per project", async () => { + const repo = new TemplateRepository(infra.db); + expect(await repo.upsertMany(P, [])).toBe(0); + expect( + await repo.upsertMany(P, [ + { + id: "welcome", + channel: "email", + topics: ["onboarding"], + content: { subject: "Hi" }, + aiPrompts: { intro: "say hi" }, + }, + { id: "otp", channel: "sms", content: { text: "{{code}}" } }, + ]), + ).toBe(2); + await repo.upsertMany(OTHER, [{ id: "welcome", channel: "push", content: {} }]); + + expect(await repo.findById(P, "welcome")).toEqual({ + id: "welcome", + channel: "email", + topics: ["onboarding"], + content: { subject: "Hi" }, + aiPrompts: { intro: "say hi" }, + }); + expect((await repo.findById(P, "otp"))!.topics).toEqual([]); + expect(await repo.findById(P, "nope")).toBeNull(); + expect((await repo.list(P)).map((t) => t.id).sort()).toEqual(["otp", "welcome"]); + + await repo.upsertMany(P, [ + { id: "welcome", channel: "email", topics: [], content: { subject: "Hello" } }, + ]); + const updated = await repo.findById(P, "welcome"); + expect(updated!.content).toEqual({ subject: "Hello" }); + expect(updated!.aiPrompts).toBeNull(); + + expect(await repo.delete(P, "welcome")).toBe(true); + expect(await repo.delete(P, "welcome")).toBe(false); + expect(await repo.findById(OTHER, "welcome")).not.toBeNull(); + }); +}); + +// ─── ProjectRepository ────────────────────────────────────────────────────── + +describe("ProjectRepository", () => { + it("lists projects newest first", async () => { + const newest = await newProject("newest"); + const list = await new ProjectRepository(infra.db).list(); + expect(list[0]!.id).toBe(newest); + expect(list.map((p) => p.id)).toEqual(expect.arrayContaining([P, OTHER])); + }); + + it("reads and updates throttle and rate-limit settings", async () => { + const repo = new ProjectRepository(infra.db); + expect(await repo.findThrottleSettings(P)).toEqual({ + throttleLimit: null, + throttleWindowHours: null, + }); + expect( + await repo.updateSettings(P, { throttleLimit: 5, throttleWindowHours: 2, rateLimitRpm: 100 }), + ).toBe(true); + expect(await repo.findThrottleSettings(P)).toEqual({ + throttleLimit: 5, + throttleWindowHours: 2, + }); + expect(await repo.updateSettings(P, { throttleLimit: null })).toBe(true); + expect((await repo.findThrottleSettings(P))!.throttleLimit).toBeNull(); + expect(await repo.findThrottleSettings(randomUUID())).toBeNull(); + expect(await repo.updateSettings(randomUUID(), { throttleLimit: 1 })).toBe(false); + }); + + it("creates, lists and deletes API keys within a project", async () => { + const repo = new ProjectRepository(infra.db); + const a = await repo.createApiKey(P, "hash-a"); + const b = await repo.createApiKey(P, "hash-b", "read_only"); + const keys = await repo.listApiKeys(P); + expect(keys.map((k) => k.id).sort()).toEqual([a.id, b.id].sort()); + expect(keys.find((k) => k.id === b.id)!.role).toBe("read_only"); + expect(keys[0]).not.toHaveProperty("keyHash"); + + expect(await repo.deleteApiKey(OTHER, a.id)).toBe(false); + expect(await repo.deleteApiKey(P, a.id)).toBe(true); + expect(await repo.listApiKeys(P)).toHaveLength(1); + await expect(repo.createApiKey(OTHER, "hash-b")).rejects.toThrow(); + }); + + it("delete removes the project and all of its data, leaving other projects alone", async () => { + const users = new UserRepository(infra.db); + const contacts = new ContactRepository(infra.db); + const repo = new ProjectRepository(infra.db); + for (const pid of [P, OTHER]) { + await users.upsertFull( + pid, + user("u", { + segments: ["s"], + preferences: { + topics: { t: false }, + channels: { sms: false }, + quietHours: [{ start: "01:00", end: "02:00" }], + }, + }), + ); + await contacts.upsert(pid, "u", "email", "u@x.com", { topics: { t: false } }); + await new TemplateRepository(infra.db).upsertMany(pid, [ + { id: "t", channel: "email", content: {} }, + ]); + await repo.createApiKey(pid, `hash-${pid}`); + await infra.sql`INSERT INTO suppressions (project_id, channel, target, reason) VALUES (${pid}, 'email', 'u@x.com', 'manual')`; + await infra.sql`INSERT INTO message_logs (project_id, task_id, channel, status) VALUES (${pid}, ${randomUUID()}, 'email', 'delivered')`; + await infra.sql`INSERT INTO workflow_definitions (project_id, name, steps) VALUES (${pid}, 'wf', '[]')`; + const [inst] = + await infra.sql`INSERT INTO workflow_instances (project_id, name) VALUES (${pid}, 'wf') RETURNING id`; + await infra.sql`INSERT INTO workflow_steps (project_id, instance_id, step_index, action) VALUES (${pid}, ${inst!.id}, '0', 'run')`; + } + + expect(await repo.delete(P)).toBe(true); + expect(await repo.delete(P)).toBe(false); + + for (const table of [ + "users", + "templates", + "project_api_keys", + "suppressions", + "message_logs", + "workflow_definitions", + "workflow_instances", + "workflow_steps", + ]) { + const { mine } = ( + await infra.sql.unsafe(`SELECT count(*)::int AS mine FROM ${table} WHERE project_id = $1`, [ + P, + ]) + )[0]!; + const { theirs } = ( + await infra.sql.unsafe( + `SELECT count(*)::int AS theirs FROM ${table} WHERE project_id = $1`, + [OTHER], + ) + )[0]!; + expect({ table, mine }).toEqual({ table, mine: 0 }); + expect({ table, theirs }).toEqual({ table, theirs: 1 }); + } + const { n } = (await infra.sql`SELECT count(*)::int AS n FROM user_contacts`)[0]!; + expect(n).toBe(1); + }); +}); + +// ─── WorkflowRepository ───────────────────────────────────────────────────── + +describe("WorkflowRepository", () => { + async function instance(pid: string, status = "pending") { + const [row] = await infra.sql` + INSERT INTO workflow_instances (project_id, name, status) VALUES (${pid}, 'wf', ${status}) RETURNING id`; + return row!.id as string; + } + + it("lists a project's definitions", async () => { + await infra.sql`INSERT INTO workflow_definitions (project_id, name, steps) VALUES (${P}, 'a', '[]'), (${OTHER}, 'b', '[]')`; + const defs = await new WorkflowRepository(infra.db).listDefinitions(P); + expect(defs.map((d) => d.name)).toEqual(["a"]); + }); + + it("returns an instance with its steps and waiters, only to its own project", async () => { + const id = await instance(P); + await infra.sql`INSERT INTO workflow_steps (project_id, instance_id, step_index, action, output) VALUES (${P}, ${id}, '0', 'notify', '{"ok":true}')`; + await infra.sql`INSERT INTO workflow_waiters (project_id, instance_id, event_name, match_criteria, expires_at) VALUES (${P}, ${id}, 'paid', '{}', now() + interval '1 hour')`; + + const repo = new WorkflowRepository(infra.db); + const got = await repo.getInstance(P, id); + expect(got.id).toBe(id); + expect(got.steps.map((s: any) => s.action)).toEqual(["notify"]); + expect(got.waiters.map((w: any) => w.eventName)).toEqual(["paid"]); + expect(await repo.getInstance(OTHER, id)).toBeNull(); + expect(await repo.getInstance(P, randomUUID())).toBeNull(); + }); + + it("cancels a pending or running instance and drops its waiters", async () => { + const repo = new WorkflowRepository(infra.db); + const pending = await instance(P, "pending"); + const running = await instance(P, "running"); + const done = await instance(P, "completed"); + await infra.sql`INSERT INTO workflow_waiters (project_id, instance_id, event_name, match_criteria, expires_at) VALUES (${P}, ${pending}, 'e', '{}', now() + interval '1 hour')`; + + expect(await repo.cancelInstance(P, pending)).toBe(true); + expect(await repo.cancelInstance(P, running)).toBe(true); + expect(await repo.cancelInstance(P, done)).toBe(false); + expect(await repo.cancelInstance(OTHER, pending)).toBe(false); + + const { status } = ( + await infra.sql`SELECT status FROM workflow_instances WHERE id = ${pending}` + )[0]!; + expect(status).toBe("canceled"); + const { n } = (await infra.sql`SELECT count(*)::int AS n FROM workflow_waiters`)[0]!; + expect(n).toBe(0); + }); +}); + +// ─── SegmentRepository ────────────────────────────────────────────────────── + +describe("SegmentRepository", () => { + it("lists a project's distinct segments", async () => { + const users = new UserRepository(infra.db); + await users.upsertManyFull(P, [ + user("a", { segments: ["vip", "beta"] }), + user("b", { segments: ["vip"] }), + ]); + await users.upsertFull(OTHER, user("c", { segments: ["secret"] })); + expect((await new SegmentRepository(infra.db).listSegments(P)).sort()).toEqual(["beta", "vip"]); + }); +}); + +// ─── AdminUserRepository ──────────────────────────────────────────────────── + +describe("AdminUserRepository", () => { + it("creates admins with a normalised email, finds them by email or username, and counts them", async () => { + const repo = new AdminUserRepository(infra.db); + expect(await repo.count()).toBe(0); + const admin = await repo.create({ + email: " Boss@Example.COM ", + username: " boss ", + passwordHash: "h1", + }); + expect(admin.email).toBe("boss@example.com"); + expect(admin.username).toBe("boss"); + expect(admin.role).toBe("admin"); + + expect((await repo.findByEmailOrUsername("BOSS@example.com"))!.id).toBe(admin.id); + expect((await repo.findByEmailOrUsername(" boss "))!.id).toBe(admin.id); + expect(await repo.findByEmailOrUsername("nobody")).toBeNull(); + expect((await repo.findById(admin.id))!.email).toBe("boss@example.com"); + expect(await repo.findById(randomUUID())).toBeNull(); + expect(await repo.count()).toBe(1); + + await expect(repo.create({ email: "boss@example.com", passwordHash: "x" })).rejects.toThrow(); + }); + + it("updates a password and never lists hashes", async () => { + const repo = new AdminUserRepository(infra.db); + const a = await repo.create({ email: "a@x.com", passwordHash: "old" }); + await repo.create({ email: "b@x.com", passwordHash: "hb", role: "viewer" }); + + expect(await repo.updatePassword(a.id, "new")).toBe(true); + expect((await repo.findById(a.id))!.passwordHash).toBe("new"); + expect(await repo.updatePassword(randomUUID(), "x")).toBe(false); + + const list = await repo.list(); + expect(list.map((u) => u.email)).toEqual(["a@x.com", "b@x.com"]); + expect(list.every((u) => !("passwordHash" in u))).toBe(true); + expect(list[1]!.role).toBe("viewer"); + }); +}); diff --git a/tests/integration/runtime.test.ts b/tests/integration/runtime.test.ts new file mode 100644 index 0000000..911e6f5 --- /dev/null +++ b/tests/integration/runtime.test.ts @@ -0,0 +1,166 @@ +import { describe, it, expect, afterEach } from "vitest"; +import { spawn } from "node:child_process"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; +import { z } from "zod"; +import { + hashPassword, + verifyPassword, + createAdminSession, + getAdminSession, + revokeAdminSession, + DUMMY_PASSWORD_HASH, +} from "@/services/auth/index.js"; +import { loadEnv, parseConfig, baseConfigSchema } from "@/config/index.js"; +import { ValidationError } from "@/shared/index.js"; +import { createDatabase, runMigrations } from "@/db/index.js"; +import { useInfra, waitFor, settle } from "./support/infra.js"; + +/** + * Process-level pieces against the real thing: admin sessions in Redis, + * configuration read from a real .env file, migrations against Postgres, and + * the server shutting down on a real signal. + */ + +const infra = useInfra(); + +describe("admin sessions", () => { + const admin = { id: randomUUID(), email: "a@x.com", username: "a", role: "admin" } as any; + + it("creates a session readable by its token, and revokes it", async () => { + const session = await createAdminSession(infra.redis, admin); + expect(session.token).toMatch(/^nk_sess_[0-9a-f]{64}$/); + expect(await getAdminSession(infra.redis, session.token)).toMatchObject({ + adminId: admin.id, + email: "a@x.com", + role: "admin", + }); + expect(await infra.redis.ttl(`notif:session:${session.token}`)).toBeGreaterThan(6 * 24 * 3600); + + await revokeAdminSession(infra.redis, session.token); + expect(await getAdminSession(infra.redis, session.token)).toBeNull(); + await revokeAdminSession(infra.redis, ""); + }); + + it("expires a session at its TTL", async () => { + const session = await createAdminSession(infra.redis, admin, 1); + await settle(1_100); + expect(await getAdminSession(infra.redis, session.token)).toBeNull(); + }); + + it("rejects tokens that are not sessions, and corrupt session data", async () => { + expect(await getAdminSession(infra.redis, "nk_live_abc")).toBeNull(); + expect(await getAdminSession(infra.redis, "")).toBeNull(); + await infra.redis.set("notif:session:nk_sess_corrupt", "{not json"); + expect(await getAdminSession(infra.redis, "nk_sess_corrupt")).toBeNull(); + await infra.redis.set( + "notif:session:nk_sess_stale", + JSON.stringify({ expiresAt: Date.now() - 1 }), + ); + expect(await getAdminSession(infra.redis, "nk_sess_stale")).toBeNull(); + expect(await infra.redis.exists("notif:session:nk_sess_stale")).toBe(0); + }); + + it("hashes passwords with a fresh salt and verifies them", async () => { + const [a, b] = await Promise.all([hashPassword("pw"), hashPassword("pw")]); + expect(a).not.toBe(b); + expect(await verifyPassword("pw", a)).toBe(true); + expect(await verifyPassword("PW", a)).toBe(false); + expect(await verifyPassword("pw", "garbage")).toBe(false); + expect(await verifyPassword("pw", DUMMY_PASSWORD_HASH)).toBe(false); + }); +}); + +describe("configuration", () => { + const keys = ["NOTIFKIT_TEST_A", "NOTIFKIT_TEST_B"]; + afterEach(() => keys.forEach((k) => delete process.env[k])); + + it("loads a .env file without overriding what the environment already set", () => { + const dir = mkdtempSync(join(tmpdir(), "nk-env-")); + try { + writeFileSync(join(dir, ".env"), "NOTIFKIT_TEST_A=from-file\nNOTIFKIT_TEST_B=from-file\n"); + process.env.NOTIFKIT_TEST_B = "from-env"; + loadEnv(join(dir, ".env")); + expect(process.env.NOTIFKIT_TEST_A).toBe("from-file"); + expect(process.env.NOTIFKIT_TEST_B).toBe("from-env"); + loadEnv(join(dir, "missing.env")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("applies defaults and rejects invalid values field by field", () => { + const cfg = parseConfig(baseConfigSchema, { ADMIN_API_KEY: " key \n", PIPELINE_FUSED: "1" }); + expect(cfg).toMatchObject({ + PORT: 3000, + WORKER_CONCURRENCY: 10, + ADMIN_API_KEY: "key", + PIPELINE_FUSED: true, + }); + + const err = (() => { + try { + parseConfig(baseConfigSchema, { PORT: "99999", REDIS_URL: "not a url" }); + } catch (e) { + return e; + } + })() as ValidationError; + expect(err).toBeInstanceOf(ValidationError); + expect(err.code).toBe("VALIDATION_ERROR"); + expect(Object.keys(err.fields!).sort()).toEqual(["PORT", "REDIS_URL"]); + expect(parseConfig(z.object({ n: z.coerce.number() }), { n: "4" })).toEqual({ n: 4 }); + }); +}); + +describe("database", () => { + it("migrations are idempotent on an up-to-date database", async () => { + const { db, sql } = createDatabase({ url: infra.dbUrl, maxConnections: 1 }); + try { + await runMigrations(db); + const [{ n }] = + (await sql`SELECT count(*)::int AS n FROM drizzle.__drizzle_migrations`) as any; + expect(n).toBeGreaterThanOrEqual(8); + const [{ canceled }] = (await sql`SELECT 'canceled'::workflow_status AS canceled`) as any; + expect(canceled).toBe("canceled"); + } finally { + await sql.end(); + } + }); +}); + +describe("process lifecycle", () => { + it.skipIf(process.platform === "win32")( + "shuts the server down gracefully on SIGTERM and exits cleanly", + async () => { + const script = ` + import { NotifkitServer } from "./src/server.ts"; + const server = new NotifkitServer({ + services: ["enricher", "engine", "delivery"], + redisUrl: process.env.REDIS_URL, databaseUrl: process.env.DATABASE_URL, + logLevel: "silent", autoMigrate: false, + }); + await server.start(); + console.log("READY"); + `; + const child = spawn("npx", ["--no-install", "tsx", "--eval", script], { + env: { ...process.env, REDIS_URL: infra.redisUrl, DATABASE_URL: infra.dbUrl }, + stdio: ["ignore", "pipe", "pipe"], + }); + let out = ""; + child.stdout.on("data", (c) => (out += c)); + const exited = new Promise((r) => child.on("exit", (code) => r(code))); + + await waitFor("server ready", () => out.includes("READY"), 60_000); + await waitFor( + "workers reporting", + async () => (await infra.redis.exists("notif:health:engine")) === 1, + 20_000, + ); + child.kill("SIGTERM"); + expect(await exited).toBe(0); + }, + 90_000, + ); +}); diff --git a/tests/integration/scheduler.test.ts b/tests/integration/scheduler.test.ts new file mode 100644 index 0000000..2db28ad --- /dev/null +++ b/tests/integration/scheduler.test.ts @@ -0,0 +1,410 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { randomUUID } from "node:crypto"; +import { + executeSchedulerPoll, + rebuildScheduledQueue, + SchedulerWorker, + SCHEDULER_SENTINEL_KEY, +} from "@/services/scheduler/main.js"; +import { createPriorityProducers } from "@/workers/bootstrap.js"; +import { StreamConsumer, PendingMessageScanner } from "@/queue/index.js"; +import { STREAMS, CONSUMER_GROUPS, registry, buildStreamEvent } from "@/contracts/index.js"; +import { createLogger } from "@/logger/index.js"; +import { ContactRepository, TemplateRepository, UserRepository } from "@/repositories/index.js"; +import { useInfra, waitFor } from "./support/infra.js"; + +/** + * The scheduler's pieces against real Redis ZSETs and a real + * scheduled_payloads table: parking a task, releasing it when due, and + * rebuilding the ZSETs from Postgres after Redis loses its data. + */ + +const infra = useInfra(); +const logger = createLogger({ name: "test", level: "silent" }); + +beforeEach(async () => { + await infra.reset(); +}); + +const shardOf = (taskId: string) => parseInt(taskId.slice(-1), 16) || 0; +const zkey = (taskId: string) => `notif:scheduled:zset:${shardOf(taskId)}`; + +async function park(taskId: string, at: number, payload: Record = {}) { + const full = { + projectId: randomUUID(), + taskId, + channel: "email", + priority: "normal", + scheduledAt: new Date(at).toISOString(), + ...payload, + }; + await infra.sql`INSERT INTO scheduled_payloads (task_id, payload) VALUES (${taskId}, ${JSON.stringify(full)}::jsonb)`; + await infra.redis.zadd( + zkey(taskId), + at, + JSON.stringify({ taskId, enrichedEventId: randomUUID(), traceId: `trace-${taskId}` }), + ); + return full; +} + +async function released(stream: string) { + return (await infra.redis.xrange(stream, "-", "+")).map(([, f]) => JSON.parse(f[1]!)); +} + +async function zsize() { + let n = 0; + for (let i = 0; i < 16; i++) n += await infra.redis.zcard(`notif:scheduled:zset:${i}`); + return n; +} + +describe("executeSchedulerPoll", () => { + it("releases due tasks onto the outbound lane for their priority, and only due ones", async () => { + const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND"); + const due = await park("task-a", Date.now() - 1_000, { priority: "critical" }); + await park("task-b", Date.now() - 1_000, { priority: "low" }); + await park("task-c", Date.now() + 60_000); + + expect(await executeSchedulerPoll(infra.redis, producers, logger, infra.db)).toBe(false); + + const critical = await released(STREAMS.OUTBOUND_CRITICAL); + expect(critical).toHaveLength(1); + expect(critical[0]).toMatchObject({ + type: "notification.dispatched", + payload: due, + metadata: { traceId: "trace-task-a", source: "scheduler" }, + }); + expect(await released(STREAMS.OUTBOUND_LOW)).toHaveLength(1); + expect(await released(STREAMS.OUTBOUND_NORMAL)).toHaveLength(0); + + const left = await infra.sql`SELECT task_id FROM scheduled_payloads ORDER BY task_id`; + expect(left.map((r) => r.task_id)).toEqual(["task-c"]); + expect(await zsize()).toBe(1); + }); + + it("drops a parked task whose payload was canceled, without sending it", async () => { + const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND"); + await park("task-x", Date.now() - 1_000); + await infra.sql`DELETE FROM scheduled_payloads`; + + await executeSchedulerPoll(infra.redis, producers, logger, infra.db); + expect(await released(STREAMS.OUTBOUND_NORMAL)).toEqual([]); + expect(await zsize()).toBe(0); + }); + + it("skips a member that is not valid JSON and releases the rest", async () => { + const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND"); + await infra.redis.zadd("notif:scheduled:zset:0", Date.now() - 1, "{not json"); + await park("task-0", Date.now() - 1_000); + await executeSchedulerPoll(infra.redis, producers, logger, infra.db); + expect(await released(STREAMS.OUTBOUND_NORMAL)).toHaveLength(1); + }); + + it("does nothing while another scheduler holds the poll lock", async () => { + const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND"); + await park("task-l", Date.now() - 1_000); + await infra.redis.set("notif:lock:scheduler:poll", "someone-else", "EX", 30); + + expect(await executeSchedulerPoll(infra.redis, producers, logger, infra.db)).toBe(false); + expect(await released(STREAMS.OUTBOUND_NORMAL)).toEqual([]); + // And it must not have released someone else's lock. + expect(await infra.redis.get("notif:lock:scheduler:poll")).toBe("someone-else"); + }); + + it("never releases one task twice when schedulers race", async () => { + const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND"); + for (let i = 0; i < 30; i++) await park(`race-${i.toString(16)}`, Date.now() - 1_000); + for (let round = 0; round < 10; round++) { + await Promise.all( + Array.from({ length: 4 }, () => + executeSchedulerPoll(infra.redis, producers, logger, infra.db), + ), + ); + } + const out = await released(STREAMS.OUTBOUND_NORMAL); + expect(out).toHaveLength(30); + expect(new Set(out.map((e) => e.payload.taskId)).size).toBe(30); + }); + + it("asks to be called again while a shard still has a backlog", async () => { + const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND"); + // Every id ends in "0", so all 150 land on one shard (100 per poll). + const pipeline = infra.redis.pipeline(); + const past = Date.now() - 1_000; + for (let i = 0; i < 150; i++) { + const taskId = `bulk-${i}-0`; + pipeline.zadd(zkey(taskId), past, JSON.stringify({ taskId, enrichedEventId: randomUUID() })); + } + await pipeline.exec(); + await infra.sql`INSERT INTO scheduled_payloads (task_id, payload) + SELECT 'bulk-' || i || '-0', jsonb_build_object('priority', 'normal') FROM generate_series(0, 149) AS i`; + + expect(await executeSchedulerPoll(infra.redis, producers, logger, infra.db)).toBe(true); + expect(await executeSchedulerPoll(infra.redis, producers, logger, infra.db)).toBe(false); + expect(await released(STREAMS.OUTBOUND_NORMAL)).toHaveLength(150); + }); + + it("does not delete a task's newer payload when it was re-parked while being released", async () => { + // Delivery throttles a released task and parks it again — new send time, + // same task id — sometimes before the poll that released it has finished. + const base = createPriorityProducers(infra.redis, logger, "OUTBOUND"); + const retryAt = Date.now() + 5_000; + const reparking = { + ...base, + normal: { + publishBatch: async (events: any[]) => { + const result = await base.normal.publishBatch(events); + for (const e of events) { + const payload = { + ...e.payload, + scheduledAt: new Date(retryAt).toISOString(), + throttleAttemptCount: 1, + }; + await infra.sql` + INSERT INTO scheduled_payloads (task_id, payload) VALUES (${e.payload.taskId}, ${JSON.stringify(payload)}::jsonb) + ON CONFLICT (task_id) DO UPDATE SET payload = EXCLUDED.payload`; + await infra.redis.zadd( + zkey(e.payload.taskId), + retryAt, + JSON.stringify({ + taskId: e.payload.taskId, + enrichedEventId: randomUUID(), + traceId: "retry", + }), + ); + } + return result; + }, + }, + }; + await park("task-r", Date.now() - 1_000); + + await executeSchedulerPoll(infra.redis, reparking, logger, infra.db); + + const rows = await infra.sql`SELECT payload FROM scheduled_payloads WHERE task_id = 'task-r'`; + expect(rows).toHaveLength(1); + expect(rows[0]!.payload.throttleAttemptCount).toBe(1); + }); + + it("keeps releasing the other shards when one cannot be read", async () => { + const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND"); + await infra.redis.set("notif:scheduled:zset:3", "wrong type"); + await park("task-4", Date.now() - 1_000); + await expect(executeSchedulerPoll(infra.redis, producers, logger, infra.db)).resolves.toBe( + false, + ); + expect((await released(STREAMS.OUTBOUND_NORMAL)).map((e) => e.payload.taskId)).toEqual([ + "task-4", + ]); + }); +}); + +/** + * A parked send was cleared by the engine when it was parked. Whatever the + * person did since — hours or days of it, for a `sendAt` or a quiet-hours + * deferral — has to stop it at release. + */ +describe("executeSchedulerPoll re-checks a parked send when releasing it", () => { + let project: string; + const users = () => new UserRepository(infra.db); + const contacts = () => new ContactRepository(infra.db); + + beforeEach(async () => { + project = randomUUID(); + await infra.sql`INSERT INTO projects (id, name) VALUES (${project}, 'p')`; + await new TemplateRepository(infra.db).upsertMany(project, [ + { id: "promo", channel: "email", content: { subject: "Sale" }, topics: ["marketing"] }, + { id: "receipt", channel: "email", content: { subject: "Receipt" } }, + ]); + await users().upsertFull(project, { userId: "u", segments: [], preferences: {} }); + await contacts().upsert(project, "u", "email", "U@X.com"); + }); + + const parkFor = (taskId: string, extra: Record = {}) => + park(taskId, Date.now() - 1_000, { + projectId: project, + recipientId: "u", + templateId: "promo", + destination: "U@X.com", + ...extra, + }); + + async function poll() { + await executeSchedulerPoll( + infra.redis, + createPriorityProducers(infra.redis, logger, "OUTBOUND"), + logger, + infra.db, + ); + return (await released(STREAMS.OUTBOUND_NORMAL)).map((e) => e.payload.taskId); + } + + async function expectDropped(taskId: string) { + expect(await poll()).toEqual([]); + expect( + await infra.sql`SELECT 1 FROM scheduled_payloads WHERE task_id = ${taskId}`, + ).toHaveLength(0); + expect(await zsize()).toBe(0); + } + + it("still releases a send nothing has changed for", async () => { + await parkFor("ok-1"); + expect(await poll()).toEqual(["ok-1"]); + }); + + it("drops a send to an address suppressed since it was parked", async () => { + await parkFor("sup-1"); + // Stored normalised, while the contact kept its original casing. + await infra.sql`INSERT INTO suppressions (project_id, channel, target, reason) VALUES (${project}, 'email', 'u@x.com', 'bounced')`; + await expectDropped("sup-1"); + }); + + it("does not let a suppression on another channel or project stop it", async () => { + await parkFor("sup-2"); + await infra.sql`INSERT INTO suppressions (project_id, channel, target, reason) VALUES (${project}, 'sms', 'u@x.com', 'manual')`; + await infra.sql`INSERT INTO suppressions (project_id, channel, target, reason) VALUES (${randomUUID()}, 'email', 'u@x.com', 'manual')`; + expect(await poll()).toEqual(["sup-2"]); + }); + + it("drops a send for a topic the user opted out of, and keeps their transactional mail", async () => { + await parkFor("topic-1"); + await parkFor("topic-2", { templateId: "receipt" }); + await users().upsertFull(project, { + userId: "u", + segments: [], + preferences: { topics: { marketing: false } }, + }); + expect(await poll()).toEqual(["topic-2"]); + }); + + it("drops a send on a channel the user switched off", async () => { + await parkFor("chan-1", { templateId: "receipt" }); + await users().upsertFull(project, { + userId: "u", + segments: [], + preferences: { channels: { email: false } }, + }); + await expectDropped("chan-1"); + }); + + it("drops a send to an address that was deactivated, or a user that was deleted", async () => { + await parkFor("gone-1"); + await contacts().deactivate(project, "u", "email", "U@X.com"); + await expectDropped("gone-1"); + + await parkFor("gone-2", { recipientId: "nobody" }); + await expectDropped("gone-2"); + }); +}); + +describe("rebuildScheduledQueue", () => { + it("re-queues every parked payload at its send time, page by page, and marks the queue intact", async () => { + const at = Date.now() + 3600_000; + for (let i = 0; i < 7; i++) await park(`rb-${i}`, at + i); + await infra.sql`INSERT INTO scheduled_payloads (task_id, payload) VALUES ('legacy-1', '{"priority":"normal"}')`; + await infra.redis.flushdb(); + + expect(await rebuildScheduledQueue(infra.redis, infra.db, logger, 3)).toBe(8); + + expect(await zsize()).toBe(8); + const score = await infra.redis.zscore( + zkey("rb-4"), + JSON.stringify({ taskId: "rb-4", enrichedEventId: undefined }), + ); + expect(Number(score)).toBe(at + 4); + // A payload with no send time is due now. + const legacy = await infra.redis.zrange(zkey("legacy-1"), "0", "-1", "WITHSCORES"); + expect(Number(legacy[1]!)).toBeLessThanOrEqual(Date.now()); + expect(await infra.redis.get(SCHEDULER_SENTINEL_KEY)).toBe("1"); + }); + + it("does not duplicate a member that survived", async () => { + await park("keep-1", Date.now() + 60_000); + await infra.redis.del(zkey("keep-1")); + await rebuildScheduledQueue(infra.redis, infra.db, logger); + await rebuildScheduledQueue(infra.redis, infra.db, logger); + expect(await zsize()).toBe(1); + }); + + it("marks an empty queue intact too", async () => { + expect(await rebuildScheduledQueue(infra.redis, infra.db, logger)).toBe(0); + expect(await infra.redis.exists(SCHEDULER_SENTINEL_KEY)).toBe(1); + }); + + it("lets the poll send a task rebuilt next to a surviving copy only once", async () => { + const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND"); + await park("dup-5", Date.now() - 1_000); + await rebuildScheduledQueue(infra.redis, infra.db, logger); // adds a trace-less copy + expect(await infra.redis.zcard(zkey("dup-5"))).toBe(2); + + await executeSchedulerPoll(infra.redis, producers, logger, infra.db); + expect(await released(STREAMS.OUTBOUND_NORMAL)).toHaveLength(1); + expect(await zsize()).toBe(0); + }); +}); + +describe("SchedulerWorker", () => { + let worker: SchedulerWorker | undefined; + afterEach(async () => { + await worker?.stop(); + worker = undefined; + }); + + it("parks each scheduled task in its shard at its send time", async () => { + const consumer = new StreamConsumer({ + redis: infra.redis, + stream: STREAMS.SCHEDULED, + group: CONSUMER_GROUPS.SCHEDULER, + consumer: "s1", + blockMs: 100, + logger, + }); + const pendingScanner = new PendingMessageScanner({ + redis: infra.redis, + stream: STREAMS.SCHEDULED, + group: CONSUMER_GROUPS.SCHEDULER, + consumer: "s1", + logger, + }); + worker = new SchedulerWorker({ + consumer, + pendingScanner, + logger, + registry, + redis: infra.redis, + }); + await worker.start(); + + const at = new Date(Date.now() + 60_000).toISOString(); + const taskId = `${randomUUID()}:abc`; + const ev = { + ...buildStreamEvent( + "notification.scheduled", + { projectId: randomUUID(), enrichedEventId: randomUUID(), taskId, scheduledAt: at }, + "engine", + "trace-9", + ), + id: randomUUID(), + timestamp: new Date().toISOString(), + }; + await infra.redis.xadd(STREAMS.SCHEDULED, "*", "data", JSON.stringify(ev)); + // An invalid payload is skipped, not retried forever. + await infra.redis.xadd( + STREAMS.SCHEDULED, + "*", + "data", + JSON.stringify({ ...ev, id: randomUUID(), payload: { taskId: 1 } }), + ); + + await waitFor("parked", async () => (await infra.redis.zcard(zkey(taskId))) === 1); + const [member, score] = await infra.redis.zrange(zkey(taskId), "0", "-1", "WITHSCORES"); + expect(JSON.parse(member!)).toMatchObject({ taskId, traceId: "trace-9" }); + expect(Number(score)).toBe(Date.parse(at)); + await waitFor( + "both acked", + async () => + Number( + ((await infra.redis.xpending(STREAMS.SCHEDULED, CONSUMER_GROUPS.SCHEDULER)) as any[])[0], + ) === 0, + ); + }); +}); diff --git a/tests/integration/support/fake-internet.ts b/tests/integration/support/fake-internet.ts new file mode 100644 index 0000000..f144730 --- /dev/null +++ b/tests/integration/support/fake-internet.ts @@ -0,0 +1,111 @@ +import http from "node:http"; +import { Agent, getGlobalDispatcher, setGlobalDispatcher, type Dispatcher } from "undici"; + +/** + * Stands in for third-party APIs at the HTTP layer. + * + * Providers keep calling `fetch("https://api.telegram.org/...")` exactly as in + * production; requests to the hosts registered here are routed to a local + * server instead, so the real request is serialised, sent, answered with a + * real status code and body, and parsed by the provider's own code. Nothing in + * the provider is replaced. + */ + +export interface RecordedRequest { + method: string; + host: string; + path: string; + headers: http.IncomingHttpHeaders; + body: string; + json(): any; +} + +export type Reply = + | { + status?: number; + json?: unknown; + text?: string; + headers?: Record; + delayMs?: number; + } + | "hang-up"; + +export type Handler = (req: RecordedRequest) => Reply | Promise; + +export interface FakeInternet { + requests: RecordedRequest[]; + /** Answers requests to `host` with `handler`, replacing any previous one. */ + on(host: string, handler: Handler): void; + close(): Promise; +} + +export async function startFakeInternet(): Promise { + const handlers = new Map(); + const requests: RecordedRequest[] = []; + + const server = http.createServer((req, res) => { + let body = ""; + req.setEncoding("utf8"); + req.on("data", (c) => (body += c)); + req.on("end", () => void respond()); + const respond = async () => { + // /__host/ + const [, , host, ...rest] = (req.url ?? "/").split("/"); + const path = `/${rest.join("/")}`; + const recorded: RecordedRequest = { + method: req.method ?? "GET", + host: host ?? "", + path, + headers: req.headers, + body, + json: () => JSON.parse(body), + }; + requests.push(recorded); + const handler = handlers.get(recorded.host); + const reply: Reply = handler + ? await handler(recorded) + : { status: 404, text: "no fake for host" }; + if (reply === "hang-up") { + req.socket.destroy(); + return; + } + if (reply.delayMs) await new Promise((r) => setTimeout(r, reply.delayMs)); + const headers: Record = { ...(reply.headers ?? {}) }; + let out = reply.text ?? ""; + if (reply.json !== undefined) { + headers["content-type"] ??= "application/json"; + out = JSON.stringify(reply.json); + } + res.writeHead(reply.status ?? 200, headers); + res.end(out); + }; + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const origin = `http://127.0.0.1:${(server.address() as any).port}`; + + const previous = getGlobalDispatcher(); + const direct = new Agent(); + class Router extends Agent { + override dispatch( + opts: Dispatcher.DispatchOptions, + handler: Dispatcher.DispatchHandler, + ): boolean { + const url = new URL(opts.path, opts.origin as string); + if (!handlers.has(url.host)) return (previous as any).dispatch(opts, handler); + return direct.dispatch({ ...opts, origin, path: `/__host/${url.host}${opts.path}` }, handler); + } + } + setGlobalDispatcher(new Router()); + + return { + requests, + on(host, handler) { + handlers.set(host, handler); + }, + async close() { + setGlobalDispatcher(previous); + await direct.close(); + await new Promise((r) => server.close(() => r())); + }, + }; +} diff --git a/tests/integration/support/global-setup.ts b/tests/integration/support/global-setup.ts new file mode 100644 index 0000000..bd1c8a3 --- /dev/null +++ b/tests/integration/support/global-setup.ts @@ -0,0 +1,76 @@ +import { PostgreSqlContainer, type StartedPostgreSqlContainer } from "@testcontainers/postgresql"; +import { RedisContainer, type StartedRedisContainer } from "@testcontainers/redis"; +import postgres from "postgres"; +import type { GlobalSetupContext } from "vitest/node"; +import { createDatabase, runMigrations } from "../../../src/db/index.js"; + +/** + * One Postgres and one Redis for the whole run. + * + * Every integration test file gets its own database, cloned from a migrated + * template (see `useInfra`), and its own Redis logical database, so files run + * in parallel without seeing each other's rows or keys. Starting a container + * per file cost several seconds each and was the main reason real-infra tests + * were rationed. + */ + +export const TEMPLATE_DB = "notifkit_template"; + +declare module "vitest" { + export interface ProvidedContext { + pgAdminUrl: string; + redisBaseUrl: string; + /** Why the containers could not start, or "" when they did. */ + infraError: string; + } +} + +let pg: StartedPostgreSqlContainer | undefined; +let redis: StartedRedisContainer | undefined; + +export default async function setup({ provide }: GlobalSetupContext) { + try { + await startContainers(provide); + } catch (err) { + // Without Docker the unit tests can still run. Each integration file then + // fails on its own with this reason, rather than every file in the run + // failing here — and rather than integration tests being skipped quietly. + if (process.env.CI) throw err; + provide("pgAdminUrl", ""); + provide("redisBaseUrl", ""); + provide("infraError", err instanceof Error ? err.message : String(err)); + } + + return async () => { + await Promise.all([pg?.stop(), redis?.stop()]); + }; +} + +async function startContainers(provide: GlobalSetupContext["provide"]) { + [pg, redis] = await Promise.all([ + new PostgreSqlContainer("postgres:16-alpine").start(), + new RedisContainer("redis:7-alpine") + .withCommand(["redis-server", "--databases", "128"]) + .start(), + ]); + + const adminUrl = pg!.getConnectionUri(); + const admin = postgres(adminUrl, { max: 1, onnotice: () => {} }); + await admin.unsafe(`CREATE DATABASE ${TEMPLATE_DB}`); + await admin.end(); + + const templateUrl = withDatabase(adminUrl, TEMPLATE_DB); + const { db, sql } = createDatabase({ url: templateUrl, maxConnections: 1 }); + await runMigrations(db); + await sql.end(); + + provide("pgAdminUrl", adminUrl); + provide("redisBaseUrl", redis!.getConnectionUrl()); + provide("infraError", ""); +} + +export function withDatabase(url: string, database: string): string { + const parsed = new URL(url); + parsed.pathname = `/${database}`; + return parsed.toString(); +} diff --git a/tests/integration/support/infra.ts b/tests/integration/support/infra.ts new file mode 100644 index 0000000..af8edc4 --- /dev/null +++ b/tests/integration/support/infra.ts @@ -0,0 +1,136 @@ +import { afterAll, beforeAll, inject } from "vitest"; +import { randomBytes } from "node:crypto"; +import postgres from "postgres"; +import { Redis } from "ioredis"; +import { drizzle } from "drizzle-orm/postgres-js"; +import type { Db, Sql } from "@/db/index.js"; +import * as schema from "@/db/schema.js"; +import { registerCustomCommands } from "@/redis/index.js"; +import { TEMPLATE_DB, withDatabase } from "./global-setup.js"; + +export interface Infra { + /** Connection string for this file's private database. */ + dbUrl: string; + /** Connection string for this file's private Redis logical database. */ + redisUrl: string; + sql: Sql; + db: Db; + /** A plain client with the Lua commands registered. */ + redis: Redis; + /** Empties every table and the Redis database. */ + reset(): Promise; +} + +/** + * Registers hooks that give the calling file a real, empty Postgres database + * (migrated) and a real, empty Redis database. The returned object is filled + * in by `beforeAll`, so read its fields inside hooks and tests, not at import. + */ +export function useInfra(): Infra { + const infra = {} as Infra; + let dbName = ""; + let adminUrl = ""; + + beforeAll(async () => { + const infraError = inject("infraError"); + if (infraError) { + throw new Error( + `Integration tests need Docker for Postgres and Redis, and it is not available: ${infraError}`, + ); + } + adminUrl = inject("pgAdminUrl"); + dbName = `t_${process.env.VITEST_POOL_ID ?? "0"}_${randomBytes(6).toString("hex")}`; + await withAdmin(adminUrl, async (admin) => { + // Two files cloning the template at the same moment can collide on its + // lock; a short retry absorbs that. + for (let attempt = 0; ; attempt++) { + try { + await admin.unsafe(`CREATE DATABASE ${dbName} TEMPLATE ${TEMPLATE_DB}`); + break; + } catch (err: any) { + if (attempt >= 20 || !String(err?.message).includes("being accessed")) throw err; + await new Promise((r) => setTimeout(r, 100 + Math.random() * 200)); + } + } + }); + + infra.dbUrl = withDatabase(adminUrl, dbName); + // The production pool's 10s statement timeout exists to shed load; on a + // test machine running the chaos suites alongside, a trivial query can wait + // that long for CPU. The code under test still gets its own pools. + const sql = postgres(infra.dbUrl, { + max: 5, + onnotice: () => {}, + connection: { statement_timeout: 60_000 as any }, + }); + const db = drizzle(sql, { schema }); + infra.sql = sql; + infra.db = db; + + const index = (Number(process.env.VITEST_POOL_ID ?? "1") % 127) + 1; + infra.redisUrl = `${inject("redisBaseUrl")}/${index}`; + infra.redis = new Redis(infra.redisUrl, { maxRetriesPerRequest: null }); + registerCustomCommands(infra.redis); + await infra.redis.flushdb(); + + infra.reset = async () => { + const tables = await sql<{ tablename: string }[]>` + SELECT tablename FROM pg_tables WHERE schemaname = 'public'`; + if (tables.length > 0) { + await sql.unsafe( + `TRUNCATE ${tables.map((t) => `"${t.tablename}"`).join(", ")} RESTART IDENTITY CASCADE`, + ); + } + await infra.redis.flushdb(); + }; + }, 120_000); + + afterAll(async () => { + infra.redis?.disconnect(); + await infra.sql?.end({ timeout: 5 }).catch(() => {}); + if (dbName) { + await withAdmin(adminUrl, (admin) => + admin.unsafe(`DROP DATABASE IF EXISTS ${dbName} WITH (FORCE)`), + ).catch(() => {}); + } + }, 60_000); + + return infra; +} + +async function withAdmin(url: string, fn: (admin: postgres.Sql) => Promise): Promise { + const admin = postgres(url, { max: 1, onnotice: () => {} }); + try { + return await fn(admin); + } finally { + await admin.end({ timeout: 5 }); + } +} + +/** Polls `condition` until it holds, failing with `what` on timeout. */ +export async function waitFor( + what: string, + condition: () => Promise | boolean, + timeoutMs = 20_000, + intervalMs = 50, +): Promise { + const deadline = Date.now() + timeoutMs; + let lastErr: unknown; + while (Date.now() < deadline) { + try { + if (await condition()) return; + } catch (err) { + lastErr = err; + } + await new Promise((r) => setTimeout(r, intervalMs)); + } + throw new Error( + `timed out after ${timeoutMs}ms waiting for: ${what}` + + (lastErr ? ` (last error: ${(lastErr as Error).message})` : ""), + ); +} + +/** Waits `ms`, for asserting that something does *not* happen. */ +export function settle(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} diff --git a/tests/integration/support/pipeline-cases.ts b/tests/integration/support/pipeline-cases.ts new file mode 100644 index 0000000..4bc6878 --- /dev/null +++ b/tests/integration/support/pipeline-cases.ts @@ -0,0 +1,794 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { randomUUID } from "node:crypto"; +import { STREAMS, buildStreamEvent } from "@/contracts/index.js"; +import { globalEmitter } from "@/shared/index.js"; +import { useInfra, waitFor, settle } from "./infra.js"; +import { + RecordingTransport, + startNotifkit, + apiClient, + type Running, + type Api, +} from "./pipeline.js"; + +/** + * The delivery guarantees notifkit makes, checked end to end: a request goes + * in through the HTTP API, and the assertion is on what the provider was + * actually asked to send — never on which internal function was called. + * + * Run twice: once with every stage hopping through Redis streams, and once + * with the fused in-process pipeline, since the two take different code paths. + */ +export function definePipelineSuite(mode: { fused: boolean }) { + const infra = useInfra(); + let app: Running; + + // "bounce…" addresses hard-bounce: permanent, so delivery fails them at once + // rather than retrying. + const email = new RecordingTransport("email", { + behaviour: (task) => + task.destination?.startsWith("bounce") + ? { success: false, error: "mailbox unavailable", retryable: false } + : { success: true, providerMessageId: `pm-${task.taskId}` }, + }); + const sms = new RecordingTransport("sms"); + const push = new RecordingTransport("push", { + behaviour: (task) => + task.destination === "dead-token" + ? { success: false, invalidToken: true } + : { success: true, providerMessageId: `push-${task.taskId}` }, + }); + // A provider that only takes one message every two seconds. + const whatsapp = new RecordingTransport("whatsapp", { limits: { limit: 1, windowSeconds: 2 } }); + + beforeAll(async () => { + app = await startNotifkit(infra, { + fusedPipeline: mode.fused, + providers: [email, sms, push, whatsapp], + env: { + RATE_LIMIT_PER_HOUR: "100000", + SEGMENT_MAX_USERS: "5", + UNSUBSCRIBE_SECRET: "an-unsubscribe-secret-for-tests", + PUBLIC_URL: "https://notify.example.com", + }, + }); + }, 120_000); + + afterAll(async () => { + await app?.stop(); + }, 60_000); + + /** A fresh project with the usual templates, so tests never see each other's data. */ + async function project() { + const p = await app.project(); + const res = await p.api("PUT", "/v1/templates", { + templates: [ + { + id: "receipt", + channel: "email", + content: { + subject: "Receipt for {{name}}", + html: "

Hi {{name}}

", + text: "Hi {{name}}, raw: {{{name}}}", + }, + }, + { + id: "news", + channel: "email", + topic: "newsletter", + content: { subject: "News", text: "Hello" }, + }, + { id: "otp", channel: "sms", content: { text: "Code {{code}}" } }, + ], + }); + expect(res.status).toBe(200); + return p; + } + + async function addUser(api: Api, body: Record) { + const res = await api("POST", "/v1/users", body); + expect(res.status).toBe(201); + } + + async function notify(api: Api, body: Record, headers?: Record) { + const res = await api("POST", "/v1/notify", body, headers); + expect(res.status).toBe(202); + return res.body; + } + + const allSent = (projectId: string) => [ + ...email.for(projectId), + ...sms.for(projectId), + ...push.for(projectId), + ...whatsapp.for(projectId), + ]; + + /** + * Nothing delivered after the pipeline has had time to act. A marker + * notification sent afterwards to a control user proves the pipeline is + * running, so an empty result is not just a slow one. + */ + async function expectNothingSentFor( + p: { id: string; api: Api }, + filter: (t: any) => boolean = () => true, + ) { + const control = `control-${randomUUID().slice(0, 6)}`; + await addUser(p.api, { id: control, email: `${control}@x.com` }); + await notify(p.api, { user: control, template: "receipt", data: { name: "c" } }); + await waitFor("control delivered", () => + email.for(p.id).some((t) => t.recipientId === control), + ); + await settle(300); + expect( + allSent(p.id) + .filter((t) => t.recipientId !== control) + .filter(filter), + ).toEqual([]); + } + + describe(`pipeline (${mode.fused ? "fused" : "streamed"})`, () => { + it("delivers one rendered email per notify, and records it", async () => { + const p = await project(); + await addUser(p.api, { id: "u1", email: "u1@x.com" }); + + await notify(p.api, { user: "u1", template: "receipt", data: { name: "\r\nBcc: x" } }); + + await waitFor("delivered", () => email.for(p.id).length === 1); + const [task] = email.for(p.id); + expect(task!.destination).toBe("u1@x.com"); + expect(task!.renderedContent.content).toEqual({ + subject: "Receipt for Bcc: x", + html: "

Hi <Al>\r\nBcc: x

", + text: "Hi \r\nBcc: x, raw: \r\nBcc: x", + }); + + await waitFor("logged", async () => { + const rows = + await infra.sql`SELECT kind, status FROM message_logs WHERE task_id = ${task!.taskId} ORDER BY kind`; + return rows.length === 2; + }); + const rows = + await infra.sql`SELECT kind, status, provider_message_id FROM message_logs WHERE task_id = ${task!.taskId} ORDER BY kind`; + expect(rows.map((r) => [r.kind, r.status])).toEqual([ + ["attempt", "delivered"], + ["dispatched", "dispatched"], + ]); + expect(rows[0]!.provider_message_id).toBe(`pm-${task!.taskId}`); + + const status = await p.api("GET", `/v1/notifications/${encodeURIComponent(task!.taskId)}`); + expect(status.status).toBe(200); + expect(status.body.status).toBe("delivered"); + + const [outbox] = + await infra.sql`SELECT provider_message_id FROM delivery_outbox WHERE task_id = ${task!.taskId}`; + expect(outbox!.provider_message_id).toBe(`pm-${task!.taskId}`); + + await settle(300); + expect(email.for(p.id)).toHaveLength(1); + }); + + it("sends once per requested channel and once per address", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: ["a@x.com", "b@x.com"], phone: "+15550001" }); + + await notify(p.api, { user: "u", template: "receipt", channels: ["email", "sms"] }); + + await waitFor("three sends", () => allSent(p.id).length === 3); + expect( + email + .for(p.id) + .map((t) => t.destination) + .sort(), + ).toEqual(["a@x.com", "b@x.com"]); + expect(sms.for(p.id).map((t) => t.destination)).toEqual(["+15550001"]); + await settle(300); + expect(allSent(p.id)).toHaveLength(3); + }); + + it("creates inline users on the fly", async () => { + const p = await project(); + await notify(p.api, { user: { id: "inline", email: "inline@x.com" }, template: "receipt" }); + await waitFor("delivered", () => email.for(p.id).length === 1); + expect(email.for(p.id)[0]!.destination).toBe("inline@x.com"); + }); + + it("reaches every member of a segment exactly once, and only in this project", async () => { + const p = await project(); + const other = await project(); + for (const id of ["s1", "s2", "s3"]) + await addUser(p.api, { id, email: `${id}@x.com`, segments: ["vip"] }); + await addUser(p.api, { id: "outsider", email: "out@x.com" }); + await addUser(other.api, { id: "foreign", email: "foreign@x.com", segments: ["vip"] }); + + await notify(p.api, { segment: "vip", template: "receipt" }); + + await waitFor("3 delivered", () => email.for(p.id).length === 3); + await settle(500); + expect( + email + .for(p.id) + .map((t) => t.recipientId) + .sort(), + ).toEqual(["s1", "s2", "s3"]); + expect(email.for(other.id)).toEqual([]); + }); + + it("reaches users subscribed to a topic", async () => { + const p = await project(); + await addUser(p.api, { + id: "fan", + email: "fan@x.com", + preferences: { topics: { launches: true } }, + }); + await addUser(p.api, { + id: "meh", + email: "meh@x.com", + preferences: { topics: { launches: false } }, + }); + + await notify(p.api, { topic: "launches", template: "receipt" }); + + await waitFor("delivered", () => email.for(p.id).length === 1); + expect(email.for(p.id)[0]!.recipientId).toBe("fan"); + }); + + it("refuses a segment larger than the fan-out limit, and says so", async () => { + const p = await project(); + for (let i = 0; i < 6; i++) + await addUser(p.api, { id: `big${i}`, email: `big${i}@x.com`, segments: ["huge"] }); + + const failures: unknown[][] = []; + const onFailed = (...args: unknown[]) => failures.push(args); + app.server.on("notification:failed", onFailed); + try { + await notify(p.api, { segment: "huge", template: "receipt" }); + await waitFor("failure reported", () => failures.length > 0); + expect(String(failures[0]![1])).toMatch(/exceeds limit/); + await expectNothingSentFor(p); + } finally { + app.server.off("notification:failed", onFailed); + } + }); + + it("sends nothing to an unknown user", async () => { + const p = await project(); + await notify(p.api, { user: "ghost", template: "receipt" }); + await expectNothingSentFor(p); + }); + + it("sends nothing for a template that does not exist", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: "u@x.com" }); + await notify(p.api, { user: "u", template: "missing" }); + await expectNothingSentFor(p); + }); + + it("sends once for repeated requests carrying the same idempotency key", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: "u@x.com" }); + const key = `order-${randomUUID()}`; + await Promise.all([ + notify(p.api, { user: "u", template: "receipt" }, { "x-idempotency-key": key }), + notify(p.api, { user: "u", template: "receipt" }, { "x-idempotency-key": key }), + notify(p.api, { user: "u", template: "receipt" }, { "x-idempotency-key": key }), + ]); + await waitFor("delivered", () => email.for(p.id).length >= 1); + await settle(1_000); + expect(email.for(p.id)).toHaveLength(1); + }); + + it("sends once when the same inbound event is read twice", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: "u@x.com" }); + const event = { + ...buildStreamEvent( + "notification.requested", + { projectId: p.id, target: { type: "user", userId: "u" }, templateId: "receipt" }, + "test", + ), + id: randomUUID(), + timestamp: new Date().toISOString(), + }; + await infra.redis.xadd(STREAMS.INBOUND_NORMAL, "*", "data", JSON.stringify(event)); + await infra.redis.xadd(STREAMS.INBOUND_NORMAL, "*", "data", JSON.stringify(event)); + + await waitFor("delivered", () => email.for(p.id).length >= 1); + await settle(1_000); + expect(email.for(p.id)).toHaveLength(1); + }); + + describe("legacy notification.created events", () => { + async function legacy(projectId: string, recipientId: string, templateId: string) { + const event = { + ...buildStreamEvent( + "notification.created", + { projectId, recipientId, channel: "email", templateId, payload: { name: "Legacy" } }, + "test", + ), + id: randomUUID(), + timestamp: new Date().toISOString(), + }; + await infra.redis.xadd(STREAMS.INBOUND_NORMAL, "*", "data", JSON.stringify(event)); + } + + it("are still delivered", async () => { + const p = await project(); + await addUser(p.api, { id: "old", email: "old@x.com" }); + await legacy(p.id, "old", "receipt"); + await waitFor("delivered", () => email.for(p.id).length === 1); + expect(email.for(p.id)[0]!.renderedContent.content).toMatchObject({ + subject: "Receipt for Legacy", + }); + }); + + it("respect the user's topic opt-outs", async () => { + const p = await project(); + await addUser(p.api, { + id: "old", + email: "old@x.com", + preferences: { topics: { newsletter: false } }, + }); + await legacy(p.id, "old", "news"); + await expectNothingSentFor(p); + }); + }); + + describe("consent", () => { + it("does not send a topic's mail to a user who opted out of that topic", async () => { + const p = await project(); + await addUser(p.api, { + id: "u", + email: "u@x.com", + preferences: { topics: { newsletter: false } }, + }); + await notify(p.api, { user: "u", template: "news" }); + await expectNothingSentFor(p); + }); + + it("still sends transactional mail to that user", async () => { + const p = await project(); + await addUser(p.api, { + id: "u", + email: "u@x.com", + preferences: { topics: { newsletter: false } }, + }); + await notify(p.api, { user: "u", template: "receipt" }); + await waitFor("delivered", () => email.for(p.id).length === 1); + }); + + it("skips only the address that opted out of the topic", async () => { + const p = await project(); + await addUser(p.api, { + id: "u", + contacts: [ + { + channel: "email", + target: "work@x.com", + preferences: { topics: { newsletter: false } }, + }, + { channel: "email", target: "home@x.com" }, + ], + }); + await notify(p.api, { user: "u", template: "news" }); + await waitFor("delivered", () => email.for(p.id).length >= 1); + await settle(500); + expect(email.for(p.id).map((t) => t.destination)).toEqual(["home@x.com"]); + }); + + it("does not use a channel the user switched off", async () => { + const p = await project(); + await addUser(p.api, { + id: "u", + email: "u@x.com", + phone: "+1555", + preferences: { channels: { sms: false } }, + }); + await notify(p.api, { user: "u", template: "receipt", channels: ["email", "sms"] }); + await waitFor("email delivered", () => email.for(p.id).length === 1); + await settle(500); + expect(sms.for(p.id)).toEqual([]); + }); + + it("never sends to a suppressed address, whatever its case, even when critical", async () => { + const p = await project(); + const res = await p.api("POST", "/v1/suppressions", { + channel: "email", + target: "Blocked@X.com", + reason: "complained", + }); + expect(res.status).toBe(201); + await addUser(p.api, { id: "u", email: "blocked@x.com" }); + await notify(p.api, { user: "u", template: "receipt", priority: "critical" }); + await expectNothingSentFor(p, (t) => t.recipientId === "u"); + }); + + it("puts a working one-click unsubscribe on topic mail, and honours it", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: "u@x.com" }); + await notify(p.api, { user: "u", template: "news" }); + await waitFor("delivered", () => email.for(p.id).length === 1); + + const headers = email.for(p.id)[0]!.deliveryOptions.headers!; + expect(headers["List-Unsubscribe-Post"]).toBe("List-Unsubscribe=One-Click"); + const url = new URL(headers["List-Unsubscribe"]!.slice(1, -1)); + expect(url.origin).toBe("https://notify.example.com"); + + // What a mail client does when the recipient presses the button. + const oneClick = await apiClient(app.baseUrl)( + "POST", + `${url.pathname}${url.search}`, + "List-Unsubscribe=One-Click", + { + "content-type": "application/x-www-form-urlencoded", + }, + ); + expect(oneClick.status).toBe(200); + + const prefs = await p.api("GET", "/v1/users/u/preferences"); + expect(prefs.body.topics).toEqual({ newsletter: false }); + }); + + it("puts no unsubscribe link on transactional mail", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: "u@x.com" }); + await notify(p.api, { user: "u", template: "receipt" }); + await waitFor("delivered", () => email.for(p.id).length === 1); + expect(email.for(p.id)[0]!.deliveryOptions.headers).toBeUndefined(); + }); + }); + + describe("throttling", () => { + it("caps non-critical sends per user at the project's limit", async () => { + const p = await project(); + const patched = await app.admin("PATCH", `/v1/projects/${p.id}`, { throttleLimit: 2 }); + expect(patched.status).toBe(200); + await addUser(p.api, { id: "u", email: "u@x.com" }); + + for (let i = 0; i < 4; i++) await notify(p.api, { user: "u", template: "receipt" }); + + await waitFor("two delivered", () => email.for(p.id).length === 2); + await settle(1_000); + expect(email.for(p.id)).toHaveLength(2); + }); + + it("lets critical sends through the per-user cap", async () => { + const p = await project(); + await app.admin("PATCH", `/v1/projects/${p.id}`, { throttleLimit: 1 }); + await addUser(p.api, { id: "u", email: "u@x.com" }); + for (let i = 0; i < 3; i++) + await notify(p.api, { user: "u", template: "receipt", priority: "critical" }); + await waitFor("three delivered", () => email.for(p.id).length === 3); + }); + + it("defers sends over a provider's rate limit instead of dropping them", async () => { + const p = await project(); + await p.api("PUT", "/v1/templates", { + templates: [{ id: "wa", channel: "whatsapp", content: { text: "hi" } }], + }); + await addUser(p.api, { id: "u", contacts: [{ channel: "whatsapp", target: "+1777" }] }); + + for (let i = 0; i < 3; i++) + await notify(p.api, { user: "u", template: "wa", channels: ["whatsapp"] }); + + await waitFor( + "all three eventually delivered", + () => whatsapp.for(p.id).length === 3, + 60_000, + ); + }, 90_000); + }); + + describe("fallback", () => { + it("moves to the next channel when the first fails", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: "bounce@x.com", phone: "+1555" }); + await notify(p.api, { + user: "u", + template: "receipt", + channels: ["email", "sms"], + fallback: true, + }); + await waitFor("sms fallback delivered", () => sms.for(p.id).length === 1); + expect(email.for(p.id)).toEqual([]); + }); + + it("counts a notification once against the user's cap, however many channels it falls back through", async () => { + const p = await project(); + await app.admin("PATCH", `/v1/projects/${p.id}`, { throttleLimit: 1 }); + await addUser(p.api, { id: "u", email: "bounce-cap@x.com", phone: "+1555" }); + + await notify(p.api, { + user: "u", + template: "receipt", + channels: ["email", "sms"], + fallback: true, + }); + await waitFor("sms fallback delivered", () => sms.for(p.id).length === 1); + + // The cap itself still holds: a second notification is over it. + await notify(p.api, { user: "u", template: "receipt", channels: ["sms"] }); + await settle(1_500); + expect(sms.for(p.id)).toHaveLength(1); + }); + + it("moves to the next channel when the user has no address on the first", async () => { + const p = await project(); + await addUser(p.api, { id: "u", phone: "+1555" }); + await notify(p.api, { + user: "u", + template: "receipt", + channels: ["email", "sms"], + fallback: true, + }); + await waitFor("sms delivered", () => sms.for(p.id).length === 1, 10_000); + }); + + it("does not use the fallback when the first channel works", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: "ok@x.com", phone: "+1555" }); + await notify(p.api, { + user: "u", + template: "receipt", + channels: ["email", "sms"], + fallback: true, + }); + await waitFor("email delivered", () => email.for(p.id).length === 1); + await settle(500); + expect(sms.for(p.id)).toEqual([]); + }); + }); + + describe("failures", () => { + it("records a failed send and dead-letters it where the project can see and replay it", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: "bounce-dlq@x.com" }); + await notify(p.api, { user: "u", template: "receipt" }); + + await waitFor("failure logged", async () => { + const rows = + await infra.sql`SELECT status FROM message_logs WHERE project_id = ${p.id} AND status = 'failed'`; + return rows.length === 1; + }); + + let dlq: any; + await waitFor("visible in the project's DLQ", async () => { + dlq = await p.api("GET", "/v1/dlq"); + return dlq.body.messages?.length === 1; + }); + const entry = dlq.body.messages[0]; + expect(entry.eventType).toBe("notification.dispatched"); + expect(entry.payload.projectId).toBe(p.id); + + // The provider recovers; replaying the entry now delivers it. + email.behaviour = (task) => ({ success: true, providerMessageId: `pm-${task.taskId}` }); + try { + const replay = await p.api("POST", "/v1/dlq/replay", { id: entry.id }); + expect(replay.status).toBe(200); + await waitFor("replayed and delivered", () => email.for(p.id).length === 1); + expect((await p.api("GET", "/v1/dlq")).body.messages).toEqual([]); + } finally { + email.behaviour = (task) => + task.destination?.startsWith("bounce") + ? { success: false, error: "mailbox unavailable", retryable: false } + : { success: true, providerMessageId: `pm-${task.taskId}` }; + } + }); + + it("deactivates a push token the provider says is dead", async () => { + const p = await project(); + await addUser(p.api, { id: "u", pushToken: ["dead-token", "live-token"] }); + await notify(p.api, { user: "u", template: "receipt", channels: ["push"] }); + + await waitFor("live token delivered", () => push.for(p.id).length === 1); + expect(push.for(p.id)[0]!.destination).toBe("live-token"); + await waitFor("dead token deactivated", async () => { + const contacts = await p.api("GET", "/v1/users/u/contacts"); + return ( + contacts.body.contacts.find((c: any) => c.target === "dead-token")?.active === false + ); + }); + }); + }); + + describe("scheduling", () => { + it("holds a scheduled send until its time, then sends it once", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: "u@x.com" }); + const sendAt = new Date(Date.now() + 3_000); + await notify(p.api, { user: "u", template: "receipt", sendAt: sendAt.toISOString() }); + + await waitFor( + "parked", + async () => + (await p.api("GET", "/v1/notifications/scheduled")).body.scheduled?.length === 1, + ); + expect(email.for(p.id)).toEqual([]); + + await waitFor("sent", () => email.for(p.id).length === 1, 20_000); + expect(Date.now()).toBeGreaterThanOrEqual(sendAt.getTime()); + await waitFor( + "unparked", + async () => + (await p.api("GET", "/v1/notifications/scheduled")).body.scheduled?.length === 0, + ); + await settle(500); + expect(email.for(p.id)).toHaveLength(1); + }, 30_000); + + it("does not send a scheduled notification that was canceled", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: "u@x.com" }); + await notify(p.api, { + user: "u", + template: "receipt", + sendAt: new Date(Date.now() + 3_000).toISOString(), + }); + + let scheduled: any[] = []; + await waitFor("parked", async () => { + scheduled = (await p.api("GET", "/v1/notifications/scheduled")).body.scheduled ?? []; + return scheduled.length === 1; + }); + const cancel = await p.api( + "DELETE", + `/v1/notifications/${encodeURIComponent(scheduled[0].taskId)}`, + ); + expect(cancel.status).toBe(200); + + await settle(8_000); + expect(email.for(p.id)).toEqual([]); + }, 30_000); + + describe("quiet hours", () => { + const hhmm = (d: Date) => d.toISOString().slice(11, 16); + const minutes = (n: number) => new Date(Date.now() + n * 60_000); + + async function parkedSendTime(p: { api: Api }): Promise { + let scheduled: any[] = []; + await waitFor("parked", async () => { + scheduled = (await p.api("GET", "/v1/notifications/scheduled")).body.scheduled ?? []; + return scheduled.length === 1; + }); + return Date.parse(scheduled[0].payload.scheduledAt); + } + + it("defers a send during the user's quiet hours until they end", async () => { + const p = await project(); + const end = minutes(60); + await addUser(p.api, { + id: "u", + email: "u@x.com", + timezone: "UTC", + preferences: { quietHours: [{ start: hhmm(minutes(-60)), end: hhmm(end) }] }, + }); + await notify(p.api, { user: "u", template: "receipt" }); + + const at = await parkedSendTime(p); + expect(Math.abs(at - end.getTime())).toBeLessThan(61_000); + expect(email.for(p.id)).toEqual([]); + }); + + it("sends critical notifications during quiet hours", async () => { + const p = await project(); + await addUser(p.api, { + id: "u", + email: "u@x.com", + timezone: "UTC", + preferences: { quietHours: [{ start: hhmm(minutes(-60)), end: hhmm(minutes(60)) }] }, + }); + await notify(p.api, { user: "u", template: "receipt", priority: "critical" }); + await waitFor("delivered", () => email.for(p.id).length === 1); + }); + + it("keeps a send scheduled for next week on next week, even if it is quiet hours now", async () => { + const p = await project(); + const nextWeek = new Date(Date.now() + 7 * 24 * 3600_000 + 3 * 3600_000); + await addUser(p.api, { + id: "u", + email: "u@x.com", + timezone: "UTC", + preferences: { quietHours: [{ start: hhmm(minutes(-60)), end: hhmm(minutes(60)) }] }, + }); + await notify(p.api, { user: "u", template: "receipt", sendAt: nextWeek.toISOString() }); + + expect(await parkedSendTime(p)).toBe(nextWeek.getTime()); + }); + + it("defers a scheduled send that would land inside quiet hours", async () => { + const p = await project(); + const quietStart = minutes(30); + const quietEnd = minutes(120); + await addUser(p.api, { + id: "u", + email: "u@x.com", + timezone: "UTC", + preferences: { quietHours: [{ start: hhmm(quietStart), end: hhmm(quietEnd) }] }, + }); + await notify(p.api, { + user: "u", + template: "receipt", + sendAt: minutes(60).toISOString(), + }); + + const at = await parkedSendTime(p); + expect(Math.abs(at - quietEnd.getTime())).toBeLessThan(61_000); + }); + }); + }); + + describe("reporting", () => { + it("attributes every message to its campaign", async () => { + const p = await project(); + for (const id of ["c1", "c2"]) + await addUser(p.api, { id, email: `${id}@x.com`, segments: ["camp"] }); + await notify(p.api, { segment: "camp", template: "receipt", campaign: "spring-sale" }); + await waitFor("two delivered", () => email.for(p.id).length === 2); + + let stats: any; + await waitFor("stats", async () => { + stats = await p.api("GET", "/v1/campaigns/spring-sale/stats"); + return stats.status === 200 && stats.body.totals.delivered === 2; + }); + expect(stats.body.totals).toMatchObject({ + sent: 2, + delivered: 2, + failed: 0, + deliveryRate: 100, + }); + + const list = await p.api("GET", "/v1/campaigns"); + expect(list.body.campaigns).toEqual([ + expect.objectContaining({ campaign: "spring-sale", messages: 2 }), + ]); + }); + + it("pages through the delivery log without skipping or repeating a message", async () => { + const p = await project(); + for (let i = 0; i < 12; i++) + await addUser(p.api, { id: `l${i}`, email: `l${i}@x.com`, segments: ["logs"] }); + // SEGMENT_MAX_USERS is 5 in this file; send in batches of users instead. + for (let i = 0; i < 12; i++) await notify(p.api, { user: `l${i}`, template: "receipt" }); + await waitFor("12 delivered", () => email.for(p.id).length === 12); + await waitFor("12 logged", async () => { + const { n } = ( + await infra.sql`SELECT count(*)::int AS n FROM message_logs WHERE project_id = ${p.id} AND status = 'delivered'` + )[0]!; + return n === 12; + }); + + const seen: string[] = []; + let cursor: string | null = null; + for (let page = 0; page < 20; page++) { + const res: any = await p.api( + "GET", + `/v1/notifications/logs?limit=5${cursor ? `&cursor=${cursor}` : ""}`, + ); + seen.push(...res.body.logs.map((l: any) => l.taskId)); + cursor = res.body.nextCursor; + if (!cursor) break; + } + expect(new Set(seen).size).toBe(12); + expect(seen).toHaveLength(12); + }); + + it("emits delivery events on the server", async () => { + const p = await project(); + await addUser(p.api, { id: "u", email: "u@x.com" }); + const delivered: unknown[][] = []; + const on = (...args: unknown[]) => delivered.push(args); + app.server.on("delivery:delivered", on); + try { + await notify(p.api, { user: "u", template: "receipt" }); + await waitFor("event", () => delivered.some((a) => a[3] === p.id)); + const [taskId, providerMessageId, channel] = delivered.find((a) => a[3] === p.id)!; + expect(channel).toBe("email"); + expect(providerMessageId).toBe(`pm-${taskId}`); + } finally { + app.server.off("delivery:delivered", on); + } + }); + }); + }); + + return { infra, app: () => app, globalEmitter }; +} diff --git a/tests/integration/support/pipeline.ts b/tests/integration/support/pipeline.ts new file mode 100644 index 0000000..b263219 --- /dev/null +++ b/tests/integration/support/pipeline.ts @@ -0,0 +1,158 @@ +// The package entry point first, as in a real install. The services import +// through this barrel while some modules also import each other directly; +// vitest's module runner snapshots `export *` at evaluation time, so loading a +// leaf module first (the API loads the queue directly) leaves the barrel +// missing exports. Node's own ESM loader binds them lazily and is unaffected. +import "@/index.js"; +import { createServer } from "node:net"; +import { NotifkitServer, type NotifkitOptions } from "@/server.js"; +import type { DeliveryResult, Transport } from "@/transport/index.js"; +import type { NotificationChannel, NotificationDispatchedPayload } from "@/contracts/index.js"; +import type { Infra } from "./infra.js"; +import { waitFor } from "./infra.js"; + +export const ADMIN_KEY = "integration-admin-key-0123456789"; + +type Behaviour = (task: NotificationDispatchedPayload) => Promise | DeliveryResult; + +/** + * A provider that records what it was asked to send. By default every send + * succeeds; `behaviour` can fail, time out or reject specific destinations. + */ +export class RecordingTransport implements Transport { + readonly sent: NotificationDispatchedPayload[] = []; + behaviour?: Behaviour; + + constructor( + readonly channel: NotificationChannel, + options: { behaviour?: Behaviour; limits?: { limit: number; windowSeconds: number } } = {}, + ) { + this.behaviour = options.behaviour; + if (options.limits) (this as any).limits = options.limits; + } + + async send(task: NotificationDispatchedPayload): Promise { + const { signal: _signal, ...copy } = task as any; + const result = this.behaviour + ? await this.behaviour(task) + : { success: true, providerMessageId: `pm-${task.taskId}` }; + if (result.success) this.sent.push(structuredClone(copy)); + return result; + } + + for(projectId: string): NotificationDispatchedPayload[] { + return this.sent.filter((t) => t.projectId === projectId); + } +} + +export async function freePort(): Promise { + return new Promise((resolve, reject) => { + const srv = createServer(); + srv.once("error", reject); + srv.listen(0, "127.0.0.1", () => { + const port = (srv.address() as any).port as number; + srv.close(() => resolve(port)); + }); + }); +} + +export interface Api { + ( + method: string, + path: string, + body?: unknown, + headers?: Record, + ): Promise<{ + status: number; + body: any; + headers: Headers; + }>; +} + +export function apiClient( + baseUrl: string, + token?: string, + defaultHeaders: Record = {}, +): Api { + return async (method, path, body, headers = {}) => { + const res = await fetch(baseUrl + path, { + method, + headers: { + ...(token ? { authorization: `Bearer ${token}` } : {}), + ...(body !== undefined ? { "content-type": "application/json" } : {}), + ...defaultHeaders, + ...headers, + }, + body: body === undefined ? undefined : typeof body === "string" ? body : JSON.stringify(body), + }); + const text = await res.text(); + let parsed: any = text; + try { + parsed = text ? JSON.parse(text) : undefined; + } catch { + // not JSON — keep the text + } + return { status: res.status, body: parsed, headers: res.headers }; + }; +} + +export interface Running { + server: NotifkitServer; + baseUrl: string; + admin: Api; + /** Creates a project through the API and returns a client authenticated as it. */ + project(name?: string): Promise<{ id: string; apiKey: string; api: Api }>; + stop(): Promise; +} + +/** + * Starts a real NotifkitServer — API and workers in this process — against the + * file's private Postgres and Redis. Environment overrides are applied before + * start, since the services read their configuration once at boot. + */ +export async function startNotifkit( + infra: Infra, + options: Partial & { env?: Record } = {}, +): Promise { + const port = await freePort(); + Object.assign(process.env, { + ADMIN_API_KEY: ADMIN_KEY, + HOST: "127.0.0.1", + LOG_FLUSH_INTERVAL_MS: "50", + ...options.env, + }); + + const { env: _env, ...serverOptions } = options; + const server = new NotifkitServer({ + services: ["api", "enricher", "engine", "scheduler", "delivery", "events"], + redisUrl: infra.redisUrl, + databaseUrl: infra.dbUrl, + logLevel: (process.env.TEST_LOG_LEVEL as any) ?? "silent", + autoMigrate: false, + port, + ...serverOptions, + }); + await server.start(); + + const baseUrl = `http://127.0.0.1:${port}`; + const services = serverOptions.services ?? ["api"]; + if (services.includes("api") || services.includes("all")) { + await waitFor("api listening", async () => (await fetch(`${baseUrl}/live`)).ok, 10_000); + } + const admin = apiClient(baseUrl, ADMIN_KEY); + + return { + server, + baseUrl, + admin, + async project(name = "test") { + const res = await admin("POST", "/v1/projects", { name }); + if (res.status !== 201) + throw new Error(`create project failed: ${res.status} ${JSON.stringify(res.body)}`); + return { id: res.body.id, apiKey: res.body.apiKey, api: apiClient(baseUrl, res.body.apiKey) }; + }, + async stop() { + await server.stop(); + }, + }; +} diff --git a/tests/integration/worker-runtime.test.ts b/tests/integration/worker-runtime.test.ts new file mode 100644 index 0000000..f3340ac --- /dev/null +++ b/tests/integration/worker-runtime.test.ts @@ -0,0 +1,666 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { randomUUID } from "node:crypto"; +import { IdempotencyGuard } from "@/idempotency/index.js"; +import { + ApiRateLimiter, + UserThrottle, + ProjectSettingsCache, + calculateTokenBatchSize, +} from "@/rate-limiter/index.js"; +import { ProviderThrottle } from "@/services/delivery/throttle.js"; +import { RedisClient } from "@/redis/index.js"; +import { + BaseWorker, + NonRetryableError, + LockHeldError, + startHealthReporter, + type ProcessResult, +} from "@/workers/index.js"; +import { + createPriorityProducers, + createStreamConsumer, + shutdownWorker, + uniqueConsumerId, +} from "@/workers/bootstrap.js"; +import { StreamConsumer, PendingMessageScanner, type StreamMessage } from "@/queue/index.js"; +import { STREAMS, CONSUMER_GROUPS, buildStreamEvent } from "@/contracts/index.js"; +import { ProjectRepository } from "@/repositories/index.js"; +import { createLogger } from "@/logger/index.js"; +import { globalEmitter } from "@/shared/index.js"; +import { useInfra, waitFor, settle } from "./support/infra.js"; + +const infra = useInfra(); +const logger = createLogger({ name: "test", level: "silent" }); + +beforeEach(async () => { + await infra.reset(); +}); + +// ─── IdempotencyGuard ──────────────────────────────────────────────────────── + +describe("IdempotencyGuard", () => { + const guard = () => + new IdempotencyGuard({ redis: infra.redis, keyPrefix: "t:idem", ttlSeconds: 60 }); + + it("grants a lease once, reports it locked to others, and completed after marking", async () => { + const g = guard(); + expect(await g.acquireLease("m1", 30)).toBe("acquired"); + expect(await g.acquireLease("m1", 30)).toBe("locked"); + await g.markProcessed("m1"); + expect(await g.acquireLease("m1", 30)).toBe("completed"); + expect(await infra.redis.ttl("t:idem:m1")).toBeGreaterThan(30); + }); + + it("gives exactly one of many concurrent callers the lease", async () => { + const g = guard(); + const results = await Promise.all(Array.from({ length: 50 }, () => g.acquireLease("race", 30))); + expect(results.filter((r) => r === "acquired")).toHaveLength(1); + expect(results.filter((r) => r === "locked")).toHaveLength(49); + }); + + it("lets a new holder in once an abandoned lease expires", async () => { + const g = guard(); + expect(await g.acquireLease("crash", 1)).toBe("acquired"); + await settle(1_100); + expect(await g.acquireLease("crash", 1)).toBe("acquired"); + }); + + it("checkAndMark is true exactly once", async () => { + const g = guard(); + const results = await Promise.all(Array.from({ length: 20 }, () => g.checkAndMark("once"))); + expect(results.filter(Boolean)).toHaveLength(1); + expect(await g.isProcessed("once")).toBe(true); + expect(await g.isProcessed("never")).toBe(false); + }); + + it("unmark clears the marker so the id can be processed again", async () => { + const g = guard(); + await g.acquireLease("again", 30); + await g.markProcessed("again"); + await g.unmark("again"); + expect(await g.isProcessed("again")).toBe(false); + expect(await g.acquireLease("again", 30)).toBe("acquired"); + }); + + it("honours a custom TTL", async () => { + const g = guard(); + await g.checkAndMark("ttl", 5); + const ttl = await infra.redis.ttl("t:idem:ttl"); + expect(ttl).toBeGreaterThan(0); + expect(ttl).toBeLessThanOrEqual(5); + }); +}); + +// ─── Rate limiting ────────────────────────────────────────────────────────── + +describe("calculateTokenBatchSize", () => { + it("scales with the limit and stays within bounds", () => { + expect(calculateTokenBatchSize(0)).toBe(0); + expect(calculateTokenBatchSize(-5)).toBe(0); + expect(calculateTokenBatchSize(10)).toBe(1); + expect(calculateTokenBatchSize(60)).toBe(5); + expect(calculateTokenBatchSize(61)).toBe(10); + expect(calculateTokenBatchSize(6_000)).toBe(500); + expect(calculateTokenBatchSize(600_000)).toBe(500); + expect(calculateTokenBatchSize(600_000, 50)).toBe(50); + }); +}); + +describe("ApiRateLimiter", () => { + it("admits exactly the limit in one window, even when requests arrive all at once", async () => { + const limiter = new ApiRateLimiter({ redis: infra.redis, windowMs: 60_000 }); + const results = await Promise.all(Array.from({ length: 200 }, () => limiter.check("p1", 50))); + expect(results.filter(Boolean)).toHaveLength(50); + expect(await limiter.check("p1", 50)).toBe(false); + }); + + it("denies everything for a non-positive limit", async () => { + const limiter = new ApiRateLimiter({ redis: infra.redis }); + expect(await limiter.check("p", 0)).toBe(false); + expect(await limiter.check("p", -1)).toBe(false); + }); + + it("keeps projects independent", async () => { + const limiter = new ApiRateLimiter({ redis: infra.redis }); + for (let i = 0; i < 5; i++) await limiter.check("a", 5); + expect(await limiter.check("a", 5)).toBe(false); + expect(await limiter.check("b", 5)).toBe(true); + }); + + it("admits again once the window has fully passed", async () => { + const limiter = new ApiRateLimiter({ redis: infra.redis, windowMs: 500 }); + // Align to the start of a bucket so the test does not straddle a boundary. + await settle(500 - (Date.now() % 500) + 5); + for (let i = 0; i < 3; i++) expect(await limiter.check("w", 3)).toBe(true); + expect(await limiter.check("w", 3)).toBe(false); + // Two windows later the previous bucket no longer weighs in. + await settle(1_000); + expect(await limiter.check("w", 3)).toBe(true); + }); + + it("fails open when Redis errors", async () => { + const broken = new RedisClient({ url: infra.redisUrl }); + await broken.disconnect(); + const limiter = new ApiRateLimiter({ redis: broken.native, logger }); + expect(await limiter.check("p", 10)).toBe(true); + }); + + it("invalidate and clear drop local leases so the next check re-reads Redis", async () => { + const limiter = new ApiRateLimiter({ redis: infra.redis }); + expect(await limiter.check("p", 600)).toBe(true); // leases a chunk locally + await infra.redis.flushdb(); + limiter.invalidate("p"); + limiter.clear(); + expect(await limiter.check("p", 600)).toBe(true); + }); +}); + +describe("ProviderThrottle", () => { + it("allows up to the provider's limit per window and says when to retry", async () => { + const throttle = new ProviderThrottle(infra.redis, logger); + const cfg = { limit: 5, windowSeconds: 60 }; + const results = await Promise.all(Array.from({ length: 8 }, () => throttle.check("sms", cfg))); + const allowed = results.filter((r) => r.allowed); + const denied = results.filter((r) => !r.allowed); + expect(allowed).toHaveLength(5); + expect(denied).toHaveLength(3); + for (const d of denied) { + expect(d.retryAfterMs).toBeGreaterThan(0); + expect(d.retryAfterMs).toBeLessThanOrEqual(60_000); + } + }); + + it("counts channels separately", async () => { + const throttle = new ProviderThrottle(infra.redis); + const cfg = { limit: 1, windowSeconds: 60 }; + expect((await throttle.check("sms", cfg)).allowed).toBe(true); + expect((await throttle.check("sms", cfg)).allowed).toBe(false); + expect((await throttle.check("email", cfg)).allowed).toBe(true); + }); +}); + +describe("UserThrottle", () => { + it("never throttles critical sends", async () => { + const t = new UserThrottle({ redis: infra.redis, maxPerHour: 1 }); + for (let i = 0; i < 5; i++) { + expect((await t.check("p", "u", "critical")).allowed).toBe(true); + } + expect(await infra.redis.exists("throttle:p:user:u")).toBe(0); + }); + + it("a project limit of 0 blocks every non-critical send", async () => { + const t = new UserThrottle({ redis: infra.redis, maxPerHour: 100 }); + expect((await t.check("p", "u", "normal", { limit: 0 })).allowed).toBe(false); + expect((await t.check("p", "u", "critical", { limit: 0 })).allowed).toBe(true); + }); + + it("uses the project's window instead of the default", async () => { + const t = new UserThrottle({ redis: infra.redis, maxPerHour: 1, windowHours: 1 }); + expect((await t.check("p", "u", "normal", { windowHours: 24 })).allowed).toBe(true); + const ttl = await infra.redis.ttl("throttle:p:user:u"); + expect(ttl).toBeGreaterThan(3600); + }); + + it("ignores nonsensical overrides and falls back to the defaults", async () => { + const t = new UserThrottle({ redis: infra.redis, maxPerHour: 2 }); + const r = await t.check("p", "u", "normal", { limit: -3, windowHours: 0 }); + expect(r.limit).toBe(2); + expect(r.allowed).toBe(true); + }); +}); + +describe("ProjectSettingsCache (backed by the real projects table)", () => { + async function project(throttleLimit: number | null) { + const id = randomUUID(); + await infra.sql`INSERT INTO projects (id, name, throttle_limit) VALUES (${id}, 'p', ${throttleLimit})`; + return id; + } + + it("loads settings, caches them, and picks up a change after invalidate", async () => { + const repo = new ProjectRepository(infra.db); + let loads = 0; + const cache = new ProjectSettingsCache((id) => { + loads++; + return repo.findThrottleSettings(id); + }); + const id = await project(7); + + expect((await cache.get(id)).throttleLimit).toBe(7); + await repo.updateSettings(id, { throttleLimit: 9 }); + expect((await cache.get(id)).throttleLimit).toBe(7); // cached + cache.invalidate(id); + expect((await cache.get(id)).throttleLimit).toBe(9); + expect(loads).toBe(2); + }); + + it("runs one query for a burst of lookups on a cold project", async () => { + const repo = new ProjectRepository(infra.db); + let loads = 0; + const cache = new ProjectSettingsCache((id) => { + loads++; + return repo.findThrottleSettings(id); + }); + const id = await project(3); + const results = await Promise.all(Array.from({ length: 25 }, () => cache.get(id))); + expect(results.every((r) => r.throttleLimit === 3)).toBe(true); + expect(loads).toBe(1); + }); + + it("treats an unknown project as having no overrides", async () => { + const repo = new ProjectRepository(infra.db); + const cache = new ProjectSettingsCache((id) => repo.findThrottleSettings(id)); + expect(await cache.get(randomUUID())).toEqual({ + throttleLimit: null, + throttleWindowHours: null, + }); + }); + + it("does not cache a failed lookup", async () => { + let fail = true; + const cache = new ProjectSettingsCache(async () => { + if (fail) throw new Error("db down"); + return { throttleLimit: 1, throttleWindowHours: null }; + }); + await expect(cache.get("p")).rejects.toThrow("db down"); + fail = false; + expect((await cache.get("p")).throttleLimit).toBe(1); + }); + + it("does not let a lookup that raced an invalidate write stale settings", async () => { + let release!: (v: any) => void; + let calls = 0; + const cache = new ProjectSettingsCache(() => { + calls++; + if (calls === 1) return new Promise((r) => (release = r)); + return Promise.resolve({ throttleLimit: 2, throttleWindowHours: null }); + }); + const stale = cache.get("p"); + cache.invalidate("p"); + release({ throttleLimit: 1, throttleWindowHours: null }); + expect((await stale).throttleLimit).toBe(1); + expect((await cache.get("p")).throttleLimit).toBe(2); + cache.clear(); + }); +}); + +// ─── RedisClient ──────────────────────────────────────────────────────────── + +describe("RedisClient", () => { + it("reports health and stops reporting it after disconnect", async () => { + const client = new RedisClient({ url: infra.redisUrl, logger }); + expect(await client.healthCheck()).toBe(true); + await client.disconnect(); + await client.disconnect(); // idempotent + expect(await client.healthCheck()).toBe(false); + }); + + it("registers the Lua commands", async () => { + const client = new RedisClient({ url: infra.redisUrl }); + await client.native.set("lock", "me"); + expect(await client.native.releaseLock("lock", "someone-else")).toBe(0); + expect(await client.native.releaseLock("lock", "me")).toBe(1); + await client.disconnect(); + }); + + it("shares one connection per URL and closes it with the last holder", async () => { + const url = `${infra.redisUrl}?shared=${randomUUID()}`; + const a = RedisClient.shared({ url }); + const b = RedisClient.shared({ url }); + expect(a.native).toBe(b.native); + + await a.disconnect(); + await a.disconnect(); // a second release must not steal b's reference + expect(await b.healthCheck()).toBe(true); + + await b.disconnect(); + expect(await b.healthCheck()).toBe(false); + + const c = RedisClient.shared({ url }); + expect(await c.healthCheck()).toBe(true); + expect(c.native).not.toBe(a.native); + await c.disconnect(); + }); +}); + +// ─── BaseWorker ───────────────────────────────────────────────────────────── + +type Behaviour = (msg: StreamMessage, attempt?: number) => Promise; + +class ScriptedWorker extends BaseWorker { + readonly seen: { n: number; attempt: number }[] = []; + inFlight = 0; + maxInFlight = 0; + constructor( + opts: ConstructorParameters[0], + public behaviour: Behaviour, + ) { + super(opts); + } + protected async process(msg: StreamMessage, attempt = 1): Promise { + this.inFlight++; + this.maxInFlight = Math.max(this.maxInFlight, this.inFlight); + try { + this.seen.push({ n: (msg.event.payload as any).n, attempt }); + return await this.behaviour(msg, attempt); + } finally { + this.inFlight--; + } + } +} + +describe("BaseWorker", () => { + const STREAM = STREAMS.ENRICHED_NORMAL; + const GROUP = CONSUMER_GROUPS.ENGINE; + const workers: BaseWorker[] = []; + + function worker( + behaviour: Behaviour, + opts: { concurrency?: number; recoveryIntervalMs?: number; maxRetriesBeforeDlq?: number } = {}, + ) { + const consumer = new StreamConsumer({ + redis: infra.redis, + stream: STREAM, + group: GROUP, + consumer: "w1", + dlqStream: STREAMS.DEAD_LETTER, + blockMs: 100, + batchSize: 20, + logger, + }); + const pendingScanner = new PendingMessageScanner({ + redis: infra.redis, + stream: STREAM, + group: GROUP, + consumer: "w1", + logger, + }); + const w = new ScriptedWorker( + { consumer, pendingScanner, logger, recoveryIntervalMs: 60_000, ...opts }, + behaviour, + ); + workers.push(w); + return w; + } + + async function publish(n: number) { + const ev = { + ...buildStreamEvent("t", { n }, "test"), + id: randomUUID(), + timestamp: new Date().toISOString(), + }; + await infra.redis.xadd(STREAM, "*", "data", JSON.stringify(ev)); + return ev; + } + + const pending = async () => Number(((await infra.redis.xpending(STREAM, GROUP)) as any[])[0]); + const dlq = async () => + (await infra.redis.xrange(STREAMS.DEAD_LETTER, "-", "+")).map((e) => JSON.parse(e[1][1]!)); + + afterEach(async () => { + await Promise.all(workers.splice(0).map((w) => w.stop())); + }); + + it("processes and acks every message, and counts them", async () => { + const w = worker(async () => {}); + await w.start(); + for (let n = 0; n < 10; n++) await publish(n); + + await waitFor("10 processed", () => w.health().processedCount === 10); + expect(await pending()).toBe(0); + const h = w.health(); + expect(h.state).toBe("running"); + expect(h.errorCount).toBe(0); + expect(h.lastProcessedAt).not.toBeNull(); + }); + + it("refuses to start twice", async () => { + const w = worker(async () => {}); + await w.start(); + await expect(w.start()).rejects.toThrow(/cannot start/); + }); + + it("never runs more than `concurrency` messages at once", async () => { + const w = worker(() => settle(30), { concurrency: 3 }); + await w.start(); + for (let n = 0; n < 15; n++) await publish(n); + await waitFor("15 processed", () => w.health().processedCount === 15); + expect(w.maxInFlight).toBeLessThanOrEqual(3); + expect(w.maxInFlight).toBeGreaterThan(1); + }); + + it("leaves a failed message pending and retries it through recovery", async () => { + let calls = 0; + const w = worker( + async () => { + if (++calls === 1) throw new Error("transient"); + }, + { recoveryIntervalMs: 100 }, + ); + await w.start(); + await publish(1); + + await waitFor("retried and processed", () => w.health().processedCount === 1, 10_000); + expect(w.seen.map((s) => s.attempt)).toEqual([1, 2]); + expect(w.health().errorCount).toBe(1); + expect(await pending()).toBe(0); + }); + + it("dead-letters a message that keeps failing, once, and reports it", async () => { + const failed: string[] = []; + const onFailed = (id: string) => failed.push(id); + globalEmitter.on("notification:failed", onFailed); + try { + const w = worker( + async () => { + throw new Error("always"); + }, + { recoveryIntervalMs: 50, maxRetriesBeforeDlq: 2 }, + ); + await w.start(); + const ev = await publish(1); + + await waitFor("dead-lettered", async () => (await dlq()).length === 1, 15_000); + await settle(300); + expect((await dlq()).map((d) => d.id)).toEqual([ev.id]); + expect(await pending()).toBe(0); + expect(w.seen.map((s) => s.attempt)).toEqual([1, 2]); + expect(failed).toHaveLength(1); + } finally { + globalEmitter.off("notification:failed", onFailed); + } + }); + + it("dead-letters a NonRetryableError immediately", async () => { + const w = worker(async () => { + throw new NonRetryableError("bad input"); + }); + await w.start(); + const ev = await publish(1); + await waitFor("dead-lettered", async () => (await dlq()).length === 1); + expect((await dlq())[0].id).toBe(ev.id); + expect(await pending()).toBe(0); + expect(w.seen).toHaveLength(1); + }); + + it("leaves a message pending, without counting an error, when the lock is held", async () => { + const w = worker(async () => { + throw new LockHeldError(); + }); + await w.start(); + await publish(1); + await waitFor("seen", () => w.seen.length === 1); + await settle(100); + expect(await pending()).toBe(1); + expect(w.health().errorCount).toBe(0); + }); + + it("leaves a message pending when process asks not to ack", async () => { + const w = worker(async () => ({ ack: false })); + await w.start(); + await publish(1); + await waitFor("seen", () => w.seen.length === 1); + await settle(100); + expect(await pending()).toBe(1); + expect(w.health().processedCount).toBe(0); + }); + + it("stop() lets in-flight messages finish and ack", async () => { + const w = worker(() => settle(300)); + await w.start(); + await publish(1); + await waitFor("started", () => w.inFlight === 1); + await w.stop(); + expect(w.health().state).toBe("stopped"); + expect(w.health().processedCount).toBe(1); + expect(await pending()).toBe(0); + }); + + it("recover() reports the pending count through health()", async () => { + const w = worker(async () => {}); + expect(w.health().pendingCount).toBeNull(); + await infra.redis.xgroup("CREATE", STREAM, GROUP, "0", "MKSTREAM"); + await w.recover(); + expect(w.health().pendingCount).toBe(0); + }); + + describe("processInline", () => { + const ev = () => ({ + ...buildStreamEvent("t", { n: 1 }, "test"), + id: randomUUID(), + timestamp: new Date().toISOString(), + }); + + it("processes and counts a handed-over event", async () => { + const w = worker(async () => {}); + await w.processInline(ev()); + expect(w.health().processedCount).toBe(1); + expect(w.seen[0]!.attempt).toBe(1); + }); + + it("dead-letters a non-retryable failure instead of throwing it upstream", async () => { + const w = worker(async () => { + throw new NonRetryableError("nope"); + }); + const e = ev(); + await w.processInline(e); + expect((await dlq()).map((d) => d.id)).toEqual([e.id]); + expect(w.health().errorCount).toBe(1); + }); + + it("swallows a held lock", async () => { + const w = worker(async () => { + throw new LockHeldError(); + }); + await expect(w.processInline(ev())).resolves.toBeUndefined(); + expect(w.health().errorCount).toBe(0); + }); + + it("rethrows a retryable failure so the upstream entry stays pending", async () => { + const w = worker(async () => { + throw new Error("boom"); + }); + await expect(w.processInline(ev())).rejects.toThrow("boom"); + }); + }); +}); + +// ─── Bootstrap helpers ────────────────────────────────────────────────────── + +describe("bootstrap helpers", () => { + it("uniqueConsumerId never repeats", () => { + const ids = new Set(Array.from({ length: 1000 }, () => uniqueConsumerId("x"))); + expect(ids.size).toBe(1000); + for (const id of ids) expect(id.startsWith("x-")).toBe(true); + }); + + it("createStreamConsumer pairs a consumer and scanner that agree", async () => { + const redis = new RedisClient({ url: infra.redisUrl }); + const { consumer, pendingScanner } = createStreamConsumer( + { redis, logger }, + { + stream: [STREAMS.OUTBOUND_NORMAL, STREAMS.OUTBOUND_LOW], + group: CONSUMER_GROUPS.DELIVERY, + consumer: "d1", + batchSize: 5, + }, + ); + await consumer.ensureGroup(); + await infra.redis.xadd( + STREAMS.OUTBOUND_LOW, + "*", + "data", + JSON.stringify({ + ...buildStreamEvent("t", {}, "s"), + id: randomUUID(), + timestamp: new Date().toISOString(), + }), + ); + const it = consumer.readBatch(); + const batch = (await it.next()).value!; + expect(batch[0]!.stream).toBe(STREAMS.OUTBOUND_LOW); + expect(await pendingScanner.getPendingCount()).toBe(1); + await consumer.nack(batch[0]!.id, batch[0]!.event, batch[0]!.stream); + expect(await infra.redis.xlen(STREAMS.DEAD_LETTER)).toBe(1); + await consumer.stop(); + await redis.disconnect(); + }); + + it("createPriorityProducers writes each lane to its own stream", async () => { + const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND"); + await producers.critical.publish(buildStreamEvent("t", {}, "s")); + await producers.normal.publish(buildStreamEvent("t", {}, "s")); + await producers.low.publish(buildStreamEvent("t", {}, "s")); + expect(await infra.redis.xlen(STREAMS.OUTBOUND_CRITICAL)).toBe(1); + expect(await infra.redis.xlen(STREAMS.OUTBOUND_NORMAL)).toBe(1); + expect(await infra.redis.xlen(STREAMS.OUTBOUND_LOW)).toBe(1); + }); + + it("shutdownWorker stops timers, the worker, and closes both connections", async () => { + const redis = new RedisClient({ url: infra.redisUrl }); + const { createDatabase } = await import("@/db/index.js"); + const { sql } = createDatabase({ url: infra.dbUrl, maxConnections: 1 }); + await sql`SELECT 1`; + let ticks = 0; + const timer = setInterval(() => ticks++, 5); + let disconnected = false; + + await shutdownWorker("test", { + logger, + timers: [timer, null], + subscriber: { disconnect: () => (disconnected = true) }, + sql, + redis, + }); + + const after = ticks; + await settle(30); + expect(ticks).toBe(after); + expect(disconnected).toBe(true); + expect(await redis.healthCheck()).toBe(false); + await expect(sql`SELECT 1`).rejects.toThrow(); + }); +}); + +describe("startHealthReporter", () => { + it("publishes the worker's health to Redis with a short TTL", async () => { + const redis = new RedisClient({ url: infra.redisUrl }); + const w = { health: () => ({ state: "running", processedCount: 3 }) } as any; + const timer = startHealthReporter("svc", w, redis, logger, 20); + try { + await waitFor("health key", async () => (await infra.redis.exists("notif:health:svc")) === 1); + const body = JSON.parse((await infra.redis.get("notif:health:svc"))!); + expect(body).toMatchObject({ + service: "svc", + redis: true, + state: "running", + processedCount: 3, + }); + const ttl = await infra.redis.ttl("notif:health:svc"); + expect(ttl).toBeGreaterThan(0); + expect(ttl).toBeLessThanOrEqual(15); + } finally { + clearInterval(timer); + await redis.disconnect(); + } + }); +}); diff --git a/tests/integration/workflows.test.ts b/tests/integration/workflows.test.ts new file mode 100644 index 0000000..ce5db60 --- /dev/null +++ b/tests/integration/workflows.test.ts @@ -0,0 +1,356 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { randomUUID } from "node:crypto"; +import { workflow } from "@/workflows/index.js"; +import { STREAMS, buildStreamEvent } from "@/contracts/index.js"; +import { useInfra, waitFor, settle } from "./support/infra.js"; +import { RecordingTransport, startNotifkit, type Running, type Api } from "./support/pipeline.js"; + +/** + * Workflows end to end: definitions and triggers go in through the API, steps + * run in the workflow worker, events come back through the events worker, and + * the assertions are on what was delivered and on the instance's recorded state. + */ + +const infra = useInfra(); +let app: Running; +const email = new RecordingTransport("email"); + +// Code-defined workflows, registered before the server starts. +const runs: Record = {}; +workflow("onboarding", async ({ step, event }) => { + await step.notify({ template: "hello", data: { name: event.user.id } }); + const paid = await step.waitForEvent("order.paid", { + timeout: "1h", + match: { userId: event.user.id }, + }); + await step.run("record", () => ({ paidAmount: paid?.amount ?? null })); + await step.notify({ template: "thanks" }); +}); +workflow("sleepy", async ({ step, event }) => { + await step.run("count", () => { + runs[event.tag] = (runs[event.tag] ?? 0) + 1; + return runs[event.tag]; + }); + await step.wait("2s"); + await step.notify({ template: "hello" }); +}); +workflow("impatient", async ({ step }) => { + const got = await step.waitForEvent("never.happens", { timeout: "1s" }); + await step.run("outcome", () => ({ timedOut: got === null })); +}); +workflow("waits-an-hour", async ({ step }) => { + await step.wait("1h"); + await step.notify({ template: "hello" }); +}); +workflow("broken", async ({ step }) => { + await step.run("explode", () => { + throw new Error("handler bug"); + }); +}); +workflow("await-go", async ({ step }) => { + const r = await step.waitForEvent("go", { timeout: "1h" }); + await step.run("after", () => ({ got: r })); +}); + +beforeAll(async () => { + app = await startNotifkit(infra, { + services: ["api", "enricher", "engine", "delivery", "scheduler", "events", "workflow"], + providers: [email], + env: { RATE_LIMIT_PER_HOUR: "100000" }, + }); +}, 120_000); + +afterAll(async () => { + await app?.stop(); +}, 60_000); + +async function project() { + const p = await app.project(); + await p.api("PUT", "/v1/templates", { + templates: [ + { id: "hello", channel: "email", content: { subject: "Hello {{name}}" } }, + { id: "thanks", channel: "email", content: { subject: "Thanks" } }, + ], + }); + for (const id of ["alice", "bob"]) { + await p.api("POST", "/v1/users", { id, email: `${id}@x.com` }); + } + return p; +} + +async function trigger(api: Api, body: Record) { + const res = await api("POST", "/v1/workflows/trigger", body); + expect(res.status).toBe(202); + return res.body.instanceId as string; +} + +async function instance(api: Api, id: string) { + const res = await api("GET", `/v1/workflows/instances/${id}`); + return res.status === 200 ? res.body : null; +} + +async function waitForStatus(api: Api, id: string, status: string, timeoutMs = 20_000) { + await waitFor( + `instance ${status}`, + async () => (await instance(api, id))?.status === status, + timeoutMs, + ); +} + +async function sendEvent(api: Api, name: string, properties: Record) { + const res = await api("POST", "/v1/events", { name, properties }); + expect(res.status).toBe(202); +} + +describe("code-defined workflows", () => { + it("notifies, sleeps on an event that matches its user, then carries on", async () => { + const p = await project(); + const id = await trigger(p.api, { name: "onboarding", user: "alice" }); + + await waitFor("welcome sent", () => email.for(p.id).length === 1); + expect(email.for(p.id)[0]!.renderedContent.content).toEqual({ subject: "Hello alice" }); + await waitForStatus(p.api, id, "pending"); + + // Someone else's payment must not wake alice's workflow. + await sendEvent(p.api, "order.paid", { userId: "bob", amount: 1 }); + await settle(1_500); + expect(email.for(p.id)).toHaveLength(1); + + await sendEvent(p.api, "order.paid", { userId: "alice", amount: 42 }); + await waitForStatus(p.api, id, "completed"); + await waitFor("thanks sent", () => email.for(p.id).length === 2); + + const inst = await instance(p.api, id); + expect(inst.steps.map((s: any) => s.action)).toEqual([ + "notify", + "waitForEvent", + "run", + "notify", + ]); + expect(inst.steps[2].output).toEqual({ paidAmount: 42 }); + expect(inst.waiters).toEqual([]); + }); + + it("records a notify step's ids as the notification that was actually sent", async () => { + const p = await project(); + const id = await trigger(p.api, { name: "onboarding", user: "alice" }); + await waitFor("sent", () => email.for(p.id).length === 1); + await waitFor("step recorded", async () => (await instance(p.api, id))?.steps.length >= 1); + + const notifyStep = (await instance(p.api, id)).steps[0]; + const task = email.for(p.id)[0]!; + // The engine derives task ids from the id of the request event. + expect(task.taskId.startsWith(`${notifyStep.output.notificationId}:`)).toBe(true); + }); + + it("files a step's delivery under its workflow instance, so the log can be filtered by it", async () => { + const p = await project(); + const id = await trigger(p.api, { name: "onboarding", user: "alice" }); + await waitFor("sent", () => email.for(p.id).length === 1); + + const byInstance = async () => + (await p.api("GET", `/v1/notifications/logs?workflowInstanceId=${id}`)).body.logs as any[]; + await waitFor("logged under the instance", async () => (await byInstance()).length === 1); + expect((await byInstance())[0]).toMatchObject({ + taskId: email.for(p.id)[0]!.taskId, + status: "delivered", + }); + + // Another instance's filter must not pick it up. + const other = (await p.api("GET", `/v1/notifications/logs?workflowInstanceId=${randomUUID()}`)) + .body.logs; + expect(other).toEqual([]); + }); + + it("wakes after a wait, without re-running completed steps", async () => { + const p = await project(); + const tag = randomUUID(); + const id = await trigger(p.api, { name: "sleepy", user: "alice", input: { tag } }); + + await waitForStatus(p.api, id, "pending"); + expect(email.for(p.id)).toEqual([]); + await waitForStatus(p.api, id, "completed", 30_000); + await waitFor("sent after waking", () => email.for(p.id).length === 1); + expect(runs[tag]).toBe(1); + }, 40_000); + + it("resumes with null when an awaited event times out", async () => { + const p = await project(); + const id = await trigger(p.api, { name: "impatient", user: "alice" }); + await waitForStatus(p.api, id, "completed", 30_000); + const inst = await instance(p.api, id); + expect(inst.steps.find((s: any) => s.action === "run").output).toEqual({ timedOut: true }); + }, 40_000); + + it("does not cut a wait short when a stray resume arrives", async () => { + const p = await project(); + const id = await trigger(p.api, { name: "waits-an-hour", user: "alice" }); + await waitForStatus(p.api, id, "pending"); + + await infra.redis.xadd( + STREAMS.WORKFLOW_INBOUND, + "*", + "data", + JSON.stringify({ + ...buildStreamEvent( + "workflow.resumed", + { + projectId: p.id, + instanceId: id, + name: "waits-an-hour", + input: { user: { id: "alice" } }, + }, + "test", + ), + id: randomUUID(), + timestamp: new Date().toISOString(), + }), + ); + await settle(2_000); + + expect((await instance(p.api, id)).status).toBe("pending"); + expect(email.for(p.id)).toEqual([]); + }); + + it("marks an instance failed when its handler throws", async () => { + const p = await project(); + const id = await trigger(p.api, { name: "broken", user: "alice" }); + await waitForStatus(p.api, id, "failed"); + }); + + it("does not lose a resume that arrives while the instance is still suspending", async () => { + const p = await project(); + // The interleaving under test: another process holds the instance (it is + // mid-way through suspending on waitForEvent) when the events worker has + // already matched the event, filled the step in, and published the resume. + const [row] = await infra.sql` + INSERT INTO workflow_instances (project_id, name, status, input) + VALUES (${p.id}, 'await-go', 'running', ${JSON.stringify({ user: { id: "alice" } })}::jsonb) RETURNING id`; + const id = row!.id as string; + await infra.sql` + INSERT INTO workflow_steps (project_id, instance_id, step_index, action, output) + VALUES (${p.id}, ${id}, '0', 'waitForEvent', ${JSON.stringify({ orderId: "o1" })}::jsonb)`; + await infra.redis.set(`lock:workflow:${id}`, "other-process", "EX", 60); + + await infra.redis.xadd( + STREAMS.WORKFLOW_INBOUND, + "*", + "data", + JSON.stringify({ + ...buildStreamEvent( + "workflow.resumed", + { projectId: p.id, instanceId: id, name: "await-go", input: { user: { id: "alice" } } }, + "events", + ), + id: randomUUID(), + timestamp: new Date().toISOString(), + }), + ); + await settle(500); + + // The other process finishes suspending and lets go. + await infra.sql`UPDATE workflow_instances SET status = 'pending' WHERE id = ${id}`; + await infra.redis.del(`lock:workflow:${id}`); + + await waitForStatus(p.api, id, "completed", 20_000); + const inst = await instance(p.api, id); + expect(inst.steps.find((s: any) => s.action === "run").output).toEqual({ + got: { orderId: "o1" }, + }); + }, 30_000); + + it("can be canceled while it sleeps, and then never resumes", async () => { + const p = await project(); + const id = await trigger(p.api, { + name: "sleepy", + user: "alice", + input: { tag: randomUUID() }, + }); + await waitForStatus(p.api, id, "pending"); + + const res = await p.api("DELETE", `/v1/workflows/instances/${id}`); + expect(res.status).toBe(204); + expect((await p.api("DELETE", `/v1/workflows/instances/${id}`)).status).toBe(400); + + await settle(8_000); + expect((await instance(p.api, id)).status).toBe("canceled"); + expect(email.for(p.id)).toEqual([]); + }, 20_000); + + it("does nothing for a workflow nobody defined", async () => { + const p = await project(); + const id = await trigger(p.api, { name: "no-such-workflow", user: "alice" }); + await settle(1_000); + expect(await instance(p.api, id)).toBeNull(); + }); + + it("keeps instances private to their project", async () => { + const p = await project(); + const other = await project(); + const id = await trigger(p.api, { name: "broken", user: "alice" }); + await waitForStatus(p.api, id, "failed"); + expect((await other.api("GET", `/v1/workflows/instances/${id}`)).status).toBe(404); + expect((await other.api("DELETE", `/v1/workflows/instances/${id}`)).status).toBe(400); + }); +}); + +describe("JSON-defined workflows", () => { + it("runs notify, wait and waitForEvent steps defined through the API", async () => { + const p = await project(); + const created = await p.api("POST", "/v1/workflows", { + name: "json-flow", + steps: [ + { action: "notify", payload: { template: "hello", data: { name: "json" } } }, + { + action: "waitForEvent", + event: "signup.done", + options: { timeout: "1h", match: { userId: "alice" } }, + }, + { action: "notify", payload: { template: "thanks", user: "bob" } }, + ], + }); + expect(created.status).toBe(201); + expect((await p.api("GET", "/v1/workflows")).body.workflows.map((w: any) => w.name)).toEqual([ + "json-flow", + ]); + + const id = await trigger(p.api, { name: "json-flow", user: "alice" }); + await waitFor("first notify", () => email.for(p.id).length === 1); + expect(email.for(p.id)[0]!.recipientId).toBe("alice"); + await waitForStatus(p.api, id, "pending"); + + await sendEvent(p.api, "signup.done", { userId: "someone-else" }); + await settle(1_500); + expect((await instance(p.api, id)).status).toBe("pending"); + + await sendEvent(p.api, "signup.done", { userId: "alice" }); + await waitForStatus(p.api, id, "completed"); + await waitFor("second notify, to the step's own target", () => email.for(p.id).length === 2); + expect(email.for(p.id)[1]!.recipientId).toBe("bob"); + }); + + it("redefining a workflow replaces its steps", async () => { + const p = await project(); + await p.api("POST", "/v1/workflows", { + name: "v", + steps: [{ action: "notify", payload: { template: "hello" } }], + }); + await p.api("POST", "/v1/workflows", { + name: "v", + steps: [{ action: "notify", payload: { template: "thanks" } }], + }); + const id = await trigger(p.api, { name: "v", user: "alice" }); + await waitForStatus(p.api, id, "completed"); + await waitFor("sent", () => email.for(p.id).length === 1); + expect(email.for(p.id)[0]!.templateId).toBe("thanks"); + }); + + it("rejects an invalid definition", async () => { + const p = await project(); + const res = await p.api("POST", "/v1/workflows", { + name: "bad", + steps: [{ action: "teleport" }], + }); + expect(res.status).toBe(400); + }); +}); diff --git a/tests/phase3.test.ts b/tests/phase3.test.ts index 98baf73..0ee7373 100644 --- a/tests/phase3.test.ts +++ b/tests/phase3.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, vi } from "vitest"; +import { globalEmitter } from "@/shared/index.js"; import { EnricherWorker } from "@/services/enricher/main.js"; import { CircuitBreaker } from "@/shared/index.js"; import { StreamConsumer } from "@/queue/index.js"; @@ -51,17 +52,21 @@ describe("Phase 3 Reliability Features", () => { }, }; - await worker.process(message as any); + const failures: unknown[][] = []; + const onFailed = (...args: unknown[]) => failures.push(args); + globalEmitter.on("notification:failed", onFailed); + try { + await worker.process(message as any); + } finally { + globalEmitter.off("notification:failed", onFailed); + } - // Verify it emitted a failure event instead of processing - expect(mockProducer.publish).toHaveBeenCalledWith( - expect.objectContaining({ - type: "notification.failed", - payload: expect.objectContaining({ - error: expect.stringContaining("exceeds limit"), - }), - }), - ); + // Reported as a failure, and nothing fanned out. (A `notification.failed` + // on the enriched stream is not something the engine can read.) + expect(failures).toHaveLength(1); + expect(String(failures[0]![1])).toContain("exceeds limit"); + expect(mockProducer.publish).not.toHaveBeenCalled(); + expect(mockProducer.publishBatch).not.toHaveBeenCalled(); }); }); diff --git a/tests/repositories.units.test.ts b/tests/repositories.units.test.ts index 9762a7a..7c6e789 100644 --- a/tests/repositories.units.test.ts +++ b/tests/repositories.units.test.ts @@ -295,6 +295,7 @@ describe("Repository mapping and guards", () => { mockDb.queueDelete([]); // suppressions mockDb.queueDelete([]); // messageLogs mockDb.queueDelete([]); // workflowInstances + mockDb.queueDelete([]); // workflowDefinitions mockDb.queueDelete([{ id: "proj-1" }]); // projects await expect(repo.delete("proj-1")).resolves.toBe(true); @@ -302,6 +303,7 @@ describe("Repository mapping and guards", () => { mockDb.queueDelete([]); // suppressions mockDb.queueDelete([]); // messageLogs mockDb.queueDelete([]); // workflowInstances + mockDb.queueDelete([]); // workflowDefinitions mockDb.queueDelete([]); // projects await expect(repo.delete("ghost")).resolves.toBe(false); }); @@ -313,8 +315,8 @@ describe("Repository mapping and guards", () => { mockDb.queueSelect([{ id: "u-1" }, { id: "u-2" }]); // 2. Child user tables deletes (6 deletes) for (let i = 0; i < 6; i++) mockDb.queueDelete([]); - // 3. Project child tables deletes (3 deletes) - for (let i = 0; i < 3; i++) mockDb.queueDelete([]); + // 3. Project child tables deletes (4 deletes) + for (let i = 0; i < 4; i++) mockDb.queueDelete([]); // 4. Final delete of project returns deleted row mockDb.queueDelete([{ id: "proj-1" }]); diff --git a/tests/worker-consumer-ids.test.ts b/tests/worker-consumer-ids.test.ts new file mode 100644 index 0000000..f6e7a02 --- /dev/null +++ b/tests/worker-consumer-ids.test.ts @@ -0,0 +1,51 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import type * as Bootstrap from "@/workers/bootstrap.js"; + +/** + * Every replica of a worker must join its consumer group under its own name. + * In containers each replica is PID 1, so a name built from the PID alone makes + * every replica the same group member. Starting a worker twice in one process + * reproduces that exactly: same PID, two starts. + * + * Startup is cut off right after the consumer is named, so nothing connects. + */ + +const seen = vi.hoisted(() => [] as string[]); + +vi.mock("@/workers/bootstrap.js", async (importOriginal) => { + const actual = await importOriginal(); + const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }; + return { + ...actual, + createWorkerRuntime: () => ({ logger, redis: { native: {} }, sql: {}, db: {} }), + createStreamConsumer: (_runtime: unknown, options: { consumer: string }) => { + seen.push(options.consumer); + throw new Error("stop after naming the consumer"); + }, + }; +}); + +import { startAiWorker } from "@/services/ai/main.js"; +import { startEventWorker } from "@/services/events/main.js"; +import { startSchedulerWorker } from "@/services/scheduler/main.js"; +import { startWorkflowWorker } from "@/services/workflow/main.js"; + +beforeEach(() => { + seen.length = 0; +}); + +describe.each([ + ["ai", startAiWorker], + ["events", startEventWorker], + ["scheduler", startSchedulerWorker], + ["workflow", startWorkflowWorker], +] as const)("%s worker", (name, start) => { + it("joins its consumer group under a name no other replica shares", async () => { + await expect(start()).rejects.toThrow("stop after naming the consumer"); + await expect(start()).rejects.toThrow("stop after naming the consumer"); + + expect(seen).toHaveLength(2); + expect(seen[0]).not.toBe(seen[1]); + for (const id of seen) expect(id.startsWith(`${name}-`)).toBe(true); + }); +}); diff --git a/tests/workers-ai.test.ts b/tests/workers-ai.test.ts index c8b0a85..cc8bd14 100644 --- a/tests/workers-ai.test.ts +++ b/tests/workers-ai.test.ts @@ -28,7 +28,11 @@ describe("AiWorker Edge Cases", () => { }; mockGenerateAiContent = vi.fn().mockResolvedValue("generated_text"); mockTemplateCache = { getCachedTemplate: vi.fn().mockResolvedValue(null) }; - mockDb = { insert: vi.fn().mockReturnThis(), values: vi.fn().mockReturnThis() }; + mockDb = { + insert: vi.fn().mockReturnThis(), + values: vi.fn().mockReturnThis(), + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + }; worker = new AiWorker({ consumer: { ack: vi.fn(), nack: vi.fn() } as any, diff --git a/tests/workers.test.ts b/tests/workers.test.ts index 3d5ffa7..0338053 100644 --- a/tests/workers.test.ts +++ b/tests/workers.test.ts @@ -179,7 +179,12 @@ describe("BaseWorker Retry and DLQ Logic", () => { await (worker as any).processWithTracking(msg); - expect(mockConsumer.nack).toHaveBeenCalledWith("msg-1", msg.event, undefined); + expect(mockConsumer.nack).toHaveBeenCalledWith( + "msg-1", + msg.event, + undefined, + expect.stringContaining("max retries exceeded"), + ); }); it("immediately moves to DLQ on attempt 1 when process throws NonRetryableError", async () => { @@ -211,7 +216,12 @@ describe("BaseWorker Retry and DLQ Logic", () => { await (worker as any).processWithTracking(msg); - expect(mockConsumer.nack).toHaveBeenCalledWith("msg-perm-1", msg.event, undefined); + expect(mockConsumer.nack).toHaveBeenCalledWith( + "msg-perm-1", + msg.event, + undefined, + "fatal: recipient account terminated", + ); }); it("processes successfully without touching Redis retry keys on happy path", async () => { @@ -307,7 +317,7 @@ describe("BaseWorker Retry and DLQ Logic", () => { }); await expect(worker.processInline(inlineEvent)).resolves.toBeUndefined(); - expect(mockConsumer.deadLetter).toHaveBeenCalledWith(inlineEvent); + expect(mockConsumer.deadLetter).toHaveBeenCalledWith(inlineEvent, expect.any(String)); expect(mockConsumer.nack).not.toHaveBeenCalled(); }); @@ -1132,7 +1142,7 @@ describe("DeliveryWorker", () => { ); }); - it("handles channel fallback by publishing to the next channel in the fallbackChain if all providers fail", async () => { + it("handles channel fallback by publishing to the next channel in the fallbackChain once its attempts are spent", async () => { const failingTransport = { send: vi.fn().mockRejectedValue(new Error("Fatal Error")) }; mockTransportRegistry.getAll.mockReturnValue([failingTransport]); @@ -1151,6 +1161,7 @@ describe("DeliveryWorker", () => { channel: "email", priority: "normal", deliveryOptions: {}, + deliveryAttemptCount: 2, // the last of the default 3 fallbackChain: ["sms", "push"], // Fallback chain provided recipient: { id: "usr-2", @@ -1378,7 +1389,9 @@ describe("DeliveryWorker", () => { { send: vi.fn().mockResolvedValue({ success: false, error: "mailbox full" }) }, ]); - await expect(worker.process(dispatched() as any)).rejects.toThrow("mailbox full"); + await expect(worker.process(dispatched({ deliveryAttemptCount: 2 }) as any)).rejects.toThrow( + "mailbox full", + ); expect(mockGlobalEmitter.emit).toHaveBeenCalledWith( "delivery:failed", "123e4567-e89b-12d3-a456-426614174001", @@ -1393,7 +1406,9 @@ describe("DeliveryWorker", () => { { send: vi.fn().mockRejectedValue(new Error("smtp down")) }, ]); - await expect(worker.process(dispatched() as any)).rejects.toThrow("smtp down"); + await expect(worker.process(dispatched({ deliveryAttemptCount: 2 }) as any)).rejects.toThrow( + "smtp down", + ); expect(worker["idempotency"].unmark).toHaveBeenCalledWith( "123e4567-e89b-12d3-a456-426614174001", ); @@ -1487,6 +1502,9 @@ describe("DeliveryWorker", () => { describe("provider rate limiting", () => { const limitedTransport = (send: any) => ({ send, limits: { limit: 5, windowSeconds: 60 } }); + /** The payload parked in Postgres, which is what the scheduler releases. */ + const parkedPayload = () => + (worker as any).db.insert.mock.results[0].value.values.mock.calls[0][0].payload; it("reschedules the task instead of sending when the provider limit is hit", async () => { const send = vi.fn(); @@ -1501,7 +1519,7 @@ describe("DeliveryWorker", () => { const envelope = mockScheduledProducer.publish.mock.calls[0]![0]; expect(envelope.type).toBe("notification.scheduled"); expect(new Date(envelope.payload.scheduledAt).getTime()).toBeGreaterThan(Date.now()); - expect(envelope.payload.throttleAttemptCount).toBe(1); + expect(parkedPayload().throttleAttemptCount).toBe(1); }); it("counts each throttled attempt so the retries cannot loop forever", async () => { @@ -1510,7 +1528,7 @@ describe("DeliveryWorker", () => { await worker.process(dispatched({ throttleAttemptCount: 2 }) as any); - expect(mockScheduledProducer.publish.mock.calls[0]![0].payload.throttleAttemptCount).toBe(3); + expect(parkedPayload().throttleAttemptCount).toBe(3); }); it("gives up once the throttled retries exceed maxAttempts", async () => { @@ -1629,9 +1647,9 @@ describe("DeliveryWorker", () => { // ── Circuit breaker ─────────────────────────────────────────────────────── /** - * The worker keys one breaker per `${channel}:${transport.constructor.name}`, - * so these transports are real classes. Two object literals would both report - * the name "Object" and quietly share a single breaker. + * The worker keys one breaker per provider: its channel and class name, + * numbered when a channel has two of one class. These transports are real + * classes so the keys read the way production ones do. */ describe("circuit breaker", () => { class FlakyTransport { @@ -1663,16 +1681,36 @@ describe("DeliveryWorker", () => { expect(flaky.send).toHaveBeenCalledTimes(5); }); - it("reports an open breaker as a delivery failure instead of letting it escape", async () => { + it("parks a task an open breaker turned away for a later attempt", async () => { + const flaky = new FlakyTransport(); + mockTransportRegistry.getAll.mockReturnValue([flaky]); + + for (let i = 0; i < 5; i++) await attempt(dispatched()); + mockScheduledProducer.publish.mockClear(); + const err = await attempt(dispatched()); + + // An open circuit says the provider is down, not that the message is bad. + expect(err).toBeUndefined(); + expect(flaky.send).toHaveBeenCalledTimes(5); + expect(mockScheduledProducer.publish).toHaveBeenCalledTimes(1); + expect(mockGlobalEmitter.emit).not.toHaveBeenCalledWith( + "delivery:failed", + expect.anything(), + "Circuit breaker is OPEN", + expect.anything(), + expect.anything(), + ); + }); + + it("reports an open breaker on a task's last attempt as a delivery failure", async () => { mockTransportRegistry.getAll.mockReturnValue([new FlakyTransport()]); for (let i = 0; i < 5; i++) await attempt(dispatched()); - const err = await attempt(dispatched()); + const err = await attempt(dispatched({ deliveryAttemptCount: 2 })); - // A short-circuit is still a delivery failure: it must reach the emitter - // and come back as NonRetryableError, or the task retries forever against - // a provider the worker has already given up on. - expect(err).toBeInstanceOf(Error); + // Out of attempts, a short-circuit is a delivery failure like any other: + // it must reach the emitter and come back as NonRetryableError. + expect(err).toBeInstanceOf(NonRetryableError); expect((err as Error).message).toBe("Circuit breaker is OPEN"); expect(mockGlobalEmitter.emit).toHaveBeenLastCalledWith( "delivery:failed", @@ -1766,6 +1804,9 @@ describe("DeliveryWorker", () => { * a retryable throw would send an already-exhausted task around again. */ describe("fallback chain exhaustion", () => { + /** A task on the last of its attempts, so a failure is final. */ + const lastAttempt = (payload: Record = {}) => + dispatched({ deliveryAttemptCount: 2, ...payload }); const allProvidersFail = () => mockTransportRegistry.getAll.mockReturnValue([ { send: vi.fn().mockRejectedValue(new Error("Provider down")) }, @@ -1774,7 +1815,7 @@ describe("DeliveryWorker", () => { it("ends the task when the only channel fails with nothing to fall back to", async () => { allProvidersFail(); - await expect(worker.process(dispatched() as any)).rejects.toBeInstanceOf(NonRetryableError); + await expect(worker.process(lastAttempt() as any)).rejects.toBeInstanceOf(NonRetryableError); expect(mockEnrichedProducers.normal.publish).not.toHaveBeenCalled(); expect(mockGlobalEmitter.emit).toHaveBeenCalledWith( @@ -1790,7 +1831,7 @@ describe("DeliveryWorker", () => { allProvidersFail(); await expect( - worker.process(dispatched({ fallbackChain: [], recipient }) as any), + worker.process(lastAttempt({ fallbackChain: [], recipient }) as any), ).rejects.toBeInstanceOf(NonRetryableError); expect(mockEnrichedProducers.normal.publish).not.toHaveBeenCalled(); @@ -1803,7 +1844,7 @@ describe("DeliveryWorker", () => { // Without one the chain is inert, so the task has to fail terminally // rather than look like it was handed on. await expect( - worker.process(dispatched({ fallbackChain: ["sms", "push"] }) as any), + worker.process(lastAttempt({ fallbackChain: ["sms", "push"] }) as any), ).rejects.toBeInstanceOf(NonRetryableError); expect(mockEnrichedProducers.normal.publish).not.toHaveBeenCalled(); @@ -1812,7 +1853,7 @@ describe("DeliveryWorker", () => { it("clears the chain on the final hop instead of passing an empty array on", async () => { allProvidersFail(); - await worker.process(dispatched({ fallbackChain: ["sms"], recipient }) as any); + await worker.process(lastAttempt({ fallbackChain: ["sms"], recipient }) as any); const published = mockEnrichedProducers.normal.publish.mock.calls[0]![0]; expect(published.payload.channel).toBe("sms"); @@ -1824,7 +1865,7 @@ describe("DeliveryWorker", () => { allProvidersFail(); // Hop one: email fails and hands the task to sms with the chain spent. - await worker.process(dispatched({ fallbackChain: ["sms"], recipient }) as any); + await worker.process(lastAttempt({ fallbackChain: ["sms"], recipient }) as any); const next = mockEnrichedProducers.normal.publish.mock.calls[0]![0].payload; expect(next.channel).toBe("sms"); @@ -1832,7 +1873,7 @@ describe("DeliveryWorker", () => { mockEnrichedProducers.normal.publish.mockClear(); await expect( worker.process( - dispatched({ + lastAttempt({ channel: next.channel, fallbackChain: next.fallbackChain, destination: "+15551234567", @@ -3265,6 +3306,7 @@ describe("DeliveryWorker", () => { channel: "email", destination: "test@example.com", priority: "normal", + deliveryAttemptCount: 2, fallbackChain: ["sms"], recipient: { phone: "+1234567890" }, }, @@ -3304,8 +3346,10 @@ describe("DeliveryWorker", () => { }, }; - // process should throw the error so the worker retry logic handles it - await expect(worker.process(msg as any)).rejects.toThrow("Circuit breaker is OPEN"); + // Parked through the scheduler for a later attempt, not failed. + await expect(worker.process(msg as any)).resolves.toBeUndefined(); + expect(mockScheduledProducer.publish).toHaveBeenCalledTimes(1); + expect(mockIdempotency.markProcessed).not.toHaveBeenCalled(); // Ensure it was NOT fallback-published expect(mockEnrichedProducers.normal.publish).not.toHaveBeenCalled(); }); @@ -3332,6 +3376,7 @@ describe("DeliveryWorker", () => { channel: "email", destination: "test@example.com", priority: "normal", + deliveryAttemptCount: 2, fallbackChain: ["sms"], recipient: { phone: "+15551234567" }, }, @@ -3436,6 +3481,7 @@ describe("DeliveryWorker", () => { event: { payload: { taskId: "task-all-fail", + deliveryAttemptCount: 2, projectId: "proj-1", enrichedEventId: "evt-1", recipientId: "usr-1", diff --git a/tests/workflows.test.ts b/tests/workflows.test.ts index cdb85cb..ca6c614 100644 --- a/tests/workflows.test.ts +++ b/tests/workflows.test.ts @@ -296,8 +296,10 @@ describe("Workflow Engine Edge Cases", () => { expect(mockDb.set).toHaveBeenLastCalledWith({ status: "completed" }); }); - it("handles step.waitForEvent timeout (returns null)", async () => { - // 3rd run: waking up from waitForEvent timeout + it("goes back to sleep when resumed before the awaited event or its timeout", async () => { + // A resume with the step still unfilled — no event recorded, no timeout — + // is spurious (a duplicate trigger, a replayed message). A real timeout + // writes { timedOut: true }; see the next test. const msg: StreamMessage = { id: "5-0", event: { @@ -332,13 +334,9 @@ describe("Workflow Engine Edge Cases", () => { await worker.process(msg); - // Should NOT have published the success notification expect(mockNotificationProducer.publish).not.toHaveBeenCalled(); - // But should have completed - expect(mockDb.update).toHaveBeenCalled(); - // Wait, mockDb.set could be called multiple times now (first running, then completed) - // We expect the LAST call to be completed - expect(mockDb.set).toHaveBeenLastCalledWith({ status: "completed" }); + // Suspended again rather than carrying on as if the wait had timed out. + expect(mockDb.set).toHaveBeenLastCalledWith({ status: "pending" }); }); it("handles step.waitForEvent when persisted step output is { timedOut: true }", async () => { diff --git a/vitest.config.ts b/vitest.config.ts index d964b80..4ed7664 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,6 +11,8 @@ export default defineConfig({ environment: "node", include: ["tests/**/*.test.ts", "packages/**/*.test.ts"], exclude: ["**/node_modules/**", "**/dist/**", "profiling/**", ".agents/**"], + // Starts the Postgres and Redis every file under tests/integration shares. + globalSetup: ["tests/integration/support/global-setup.ts"], testTimeout: 1200_000, hookTimeout: 1200_000, coverage: {